{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2a8e89d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/PyPDF2/__init__.py:21: DeprecationWarning: PyPDF2 is deprecated. Please move to the pypdf library instead.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from evoagentx.agents.agent_manager import AgentManager\n",
    "from evoagentx.benchmark import HotPotQA\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.core.logging import logger\n",
    "from evoagentx.evaluators import Evaluator\n",
    "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
    "from evoagentx.optimizers import TextGradOptimizer\n",
    "from evoagentx.prompts import StringTemplate\n",
    "from evoagentx.workflow import SequentialWorkFlowGraph\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from evoagentx.agents.agent_manager import AgentManager\n",
    "from evoagentx.benchmark import MBPP\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.core.logging import logger\n",
    "from evoagentx.evaluators import Evaluator\n",
    "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
    "from evoagentx.optimizers import TextGradOptimizer\n",
    "from evoagentx.prompts import StringTemplate\n",
    "from evoagentx.workflow import SequentialWorkFlowGraph\n",
    "\n",
    "from evoagentx.models import OpenAILLMConfig, OpenAILLM\n",
    "from evoagentx.workflow import SEWWorkFlowGraph, STRUCTUREWorkFlowGraph\n",
    "from evoagentx.agents import AgentManager\n",
    "from evoagentx.benchmark import HumanEval,AFlowMBPP\n",
    "from evoagentx.evaluators import Evaluator \n",
    "from evoagentx.optimizers import SEWOptimizer, STRUCTUREOptimizer\n",
    "from evoagentx.optimizers.structure_optimizer import STRUCTUREWorkFlowScheme\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "\n",
    "from evoagentx.models import OpenAILLMConfig, OpenAILLM,AzureOpenAIConfig,LiteLLMConfig,LiteLLM\n",
    "from evoagentx.workflow import SEWWorkFlowGraph \n",
    "from evoagentx.agents import AgentManager\n",
    "from evoagentx.benchmark import MBPPPLUS, AFlowMBPPPLUS\n",
    "from evoagentx.evaluators import Evaluator \n",
    "from evoagentx.optimizers import SEWOptimizer \n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.benchmark import HumanEvalPLUS\n",
    "from evoagentx.benchmark import SciCode\n",
    "from copy import deepcopy\n",
    "\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "54f40417",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from evoagentx.agents.agent_manager import AgentManager\n",
    "from evoagentx.benchmark import HotPotQA\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.core.logging import logger\n",
    "from evoagentx.evaluators import Evaluator\n",
    "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
    "from evoagentx.optimizers import TextGradOptimizer\n",
    "from evoagentx.prompts import StringTemplate\n",
    "from evoagentx.workflow import SequentialWorkFlowGraph\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from evoagentx.agents.agent_manager import AgentManager\n",
    "from evoagentx.benchmark import MBPP\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.core.logging import logger\n",
    "from evoagentx.evaluators import Evaluator\n",
    "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
    "from evoagentx.optimizers import TextGradOptimizer\n",
    "from evoagentx.prompts import StringTemplate\n",
    "from evoagentx.workflow import SequentialWorkFlowGraph\n",
    "\n",
    "from evoagentx.models import OpenAILLMConfig, OpenAILLM\n",
    "from evoagentx.workflow import SEWWorkFlowGraph, STRUCTUREWorkFlowGraph\n",
    "from evoagentx.agents import AgentManager\n",
    "from evoagentx.benchmark import HumanEval,AFlowMBPP\n",
    "from evoagentx.evaluators import Evaluator \n",
    "from evoagentx.optimizers import SEWOptimizer, STRUCTUREOptimizer\n",
    "from evoagentx.optimizers.structure_optimizer import STRUCTUREWorkFlowScheme\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "\n",
    "from evoagentx.models import OpenAILLMConfig, OpenAILLM,AzureOpenAIConfig,LiteLLMConfig,LiteLLM\n",
    "from evoagentx.workflow import SEWWorkFlowGraph \n",
    "from evoagentx.agents import AgentManager\n",
    "from evoagentx.benchmark import MBPPPLUS, AFlowMBPPPLUS\n",
    "from evoagentx.evaluators import Evaluator \n",
    "from evoagentx.optimizers import SEWOptimizer \n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.benchmark import HumanEvalPLUS\n",
    "from evoagentx.benchmark import SciCode\n",
    "from evoagentx.benchmark import PertQA\n",
    "from copy import deepcopy\n",
    "\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "54fa1aa5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from evoagentx.benchmark import PertQA\n",
    "from copy import deepcopy\n",
    "\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()\n",
    "\n",
    "def collate_func(example: dict) -> dict:\n",
    "    problem = \"Question: {}\\n\\nAnswer:\".format(example[\"question\"])\n",
    "    return {\"problem\": problem}\n",
    "\n",
    "\n",
    "hotpotqa_graph_data = {\n",
    "    \"goal\": \"Answer the question based on the context. The answer should be a direct response to the question, without including explanations or reasoning.\",\n",
    "    \"tasks\": [\n",
    "        {\n",
    "            \"name\": \"answer_generate\",\n",
    "            \"description\": \"Answer the question based on the context.\",\n",
    "            \"inputs\": [\n",
    "                {\"name\": \"problem\", \"type\": \"str\", \"required\": True, \"description\": \"The problem to solve.\"}\n",
    "            ],\n",
    "            \"outputs\": [\n",
    "                {\"name\": \"answer\", \"type\": \"str\", \"required\": True, \"description\": \"The answer to the problem.\"}\n",
    "            ],\n",
    "            \"prompt_template\": StringTemplate(instruction=\"Think step by step to answer the question. You should explain your thinking process in the 'thought' field, and provide the final answer in the 'answer' field.\\nFormat your output in xml format, such as <thought>xxx</thought> and <answer>xxx</answer>.\"),\n",
    "            \"parse_mode\": \"xml\"\n",
    "        }\n",
    "    ] \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1ebace55",
   "metadata": {},
   "outputs": [],
   "source": [
    "api_key = \"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\"\n",
    "# Function to encode the image\n",
    "import os\n",
    "os.environ[\"OPENAI_API_KEY\"] = api_key\n",
    "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n",
    "\n",
    "\n",
    "llm_config = OpenAILLMConfig(model=\"gpt-4o-mini-2024-07-18\", openai_key=OPENAI_API_KEY, top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
    "llm = OpenAILLM(config=llm_config)\n",
    "executor_llm = OpenAILLM(config=llm_config)\n",
    "optimizer_llm = OpenAILLM(config=llm_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "20e078fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-03 18:43:14.547\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/norman_train.json ...\u001b[0m\n",
      "\u001b[32m2026-01-03 18:43:14.548\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/norman_train.json ...\u001b[0m\n",
      "\u001b[32m2026-01-03 18:43:14.550\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/norman_test.json ...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "benchmark = PertQA(pertdata='norman')\n",
    "workflow_graph = SequentialWorkFlowGraph.from_dict(hotpotqa_graph_data)\n",
    "agent_manager = AgentManager()\n",
    "agent_manager.add_agents_from_workflow(workflow_graph, executor_llm.config)\n",
    "\n",
    "evaluator = Evaluator(\n",
    "    llm=executor_llm, \n",
    "    agent_manager=agent_manager, \n",
    "    collate_func=collate_func, \n",
    "    num_workers=20, \n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "textgrad_optimizer = TextGradOptimizer(\n",
    "    graph=workflow_graph, \n",
    "    optimize_mode=\"all\",\n",
    "    executor_llm=executor_llm, \n",
    "    optimizer_llm=optimizer_llm,\n",
    "    batch_size=3,\n",
    "    max_steps=20,\n",
    "    evaluator=evaluator,\n",
    "    eval_every_n_steps=1,\n",
    "    eval_rounds=1,\n",
    "    save_interval=None,\n",
    "    save_path=\"./\",\n",
    "    rollback=True,\n",
    "    constraints=[]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b485d4ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "150"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(benchmark._dev_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "78d5904e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# len(benchmark._fulldata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d686ee20",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:10:35.390\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m1\u001b[0m - \u001b[1mEvaluating workflow on test set...\u001b[0m\n",
      "Evaluating workflow:   0%|          | 2/2500 [00:03<52:49,  1.27s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 5/2500 [00:03<16:22,  2.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 13/2500 [00:03<05:05,  8.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 16/2500 [00:04<04:34,  9.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 18/2500 [00:04<05:10,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 20/2500 [00:05<09:48,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 21/2500 [00:06<12:37,  3.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 23/2500 [00:06<11:13,  3.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 24/2500 [00:07<12:26,  3.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 28/2500 [00:07<08:10,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 31/2500 [00:07<06:48,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 34/2500 [00:08<05:00,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 36/2500 [00:08<04:28,  9.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 38/2500 [00:08<06:37,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 42/2500 [00:10<10:42,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 44/2500 [00:10<09:08,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 46/2500 [00:10<07:13,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 50/2500 [00:11<06:17,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 52/2500 [00:11<05:46,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 56/2500 [00:12<06:57,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 59/2500 [00:13<06:38,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 61/2500 [00:13<06:33,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 62/2500 [00:13<08:53,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 64/2500 [00:14<09:25,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 67/2500 [00:14<06:12,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 72/2500 [00:15<04:24,  9.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 74/2500 [00:15<06:45,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 75/2500 [00:16<09:03,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 78/2500 [00:16<07:13,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 79/2500 [00:16<08:57,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 81/2500 [00:17<08:25,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 83/2500 [00:17<07:18,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 84/2500 [00:18<09:17,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 86/2500 [00:18<07:10,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 87/2500 [00:18<08:45,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 89/2500 [00:18<07:53,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 93/2500 [00:19<05:31,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 97/2500 [00:19<05:03,  7.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 99/2500 [00:20<06:36,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 101/2500 [00:20<05:44,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 103/2500 [00:21<07:34,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 105/2500 [00:21<08:18,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 108/2500 [00:21<05:28,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 109/2500 [00:22<05:56,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 111/2500 [00:22<06:45,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 113/2500 [00:22<06:12,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 116/2500 [00:23<05:34,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 119/2500 [00:23<04:45,  8.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 120/2500 [00:23<06:20,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 122/2500 [00:24<08:06,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 123/2500 [00:24<11:08,  3.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 127/2500 [00:25<07:05,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 130/2500 [00:25<06:14,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 133/2500 [00:26<06:08,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 135/2500 [00:26<05:23,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 136/2500 [00:26<05:32,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 137/2500 [00:26<07:05,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 138/2500 [00:27<10:52,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 141/2500 [00:28<08:18,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 144/2500 [00:28<07:33,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 148/2500 [00:29<06:32,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 152/2500 [00:30<07:44,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 154/2500 [00:30<07:18,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 157/2500 [00:30<05:33,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 158/2500 [00:31<07:36,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 160/2500 [00:31<09:54,  3.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 164/2500 [00:32<05:28,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 166/2500 [00:32<04:58,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 167/2500 [00:32<08:14,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 169/2500 [00:33<07:16,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 173/2500 [00:33<05:22,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 174/2500 [00:33<07:38,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 176/2500 [00:34<07:13,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 178/2500 [00:34<08:26,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 180/2500 [00:35<06:20,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 183/2500 [00:35<05:28,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 185/2500 [00:36<08:15,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 188/2500 [00:36<05:56,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 190/2500 [00:36<05:31,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 194/2500 [00:37<03:29, 11.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 198/2500 [00:37<04:37,  8.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 200/2500 [00:38<08:26,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 201/2500 [00:38<09:01,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 203/2500 [00:39<08:17,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 205/2500 [00:39<06:09,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 207/2500 [00:39<05:57,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 208/2500 [00:40<07:57,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 210/2500 [00:40<06:35,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 211/2500 [00:40<06:59,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▊         | 214/2500 [00:41<06:26,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 215/2500 [00:41<07:00,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 216/2500 [00:41<07:47,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 217/2500 [00:41<07:46,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 219/2500 [00:42<07:45,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 220/2500 [00:42<10:54,  3.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 221/2500 [00:43<20:09,  1.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 222/2500 [00:44<20:17,  1.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 223/2500 [00:45<21:45,  1.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 224/2500 [00:45<17:45,  2.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 228/2500 [00:45<09:32,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 230/2500 [00:46<09:20,  4.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 233/2500 [00:46<07:30,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 234/2500 [00:46<07:34,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 235/2500 [00:47<10:16,  3.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 236/2500 [00:48<12:56,  2.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 237/2500 [00:48<16:40,  2.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 239/2500 [00:49<11:34,  3.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 243/2500 [00:49<07:00,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 244/2500 [00:49<07:15,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 245/2500 [00:50<09:40,  3.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 247/2500 [00:50<09:49,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 249/2500 [00:51<09:46,  3.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 251/2500 [00:51<10:35,  3.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 253/2500 [00:52<08:07,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 254/2500 [00:52<11:11,  3.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 256/2500 [00:53<09:42,  3.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 261/2500 [00:53<05:10,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 264/2500 [00:54<05:42,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 267/2500 [00:54<05:52,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 268/2500 [00:54<05:49,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 269/2500 [00:55<06:37,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 271/2500 [00:55<05:29,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 273/2500 [00:55<08:01,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 275/2500 [00:56<06:51,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 276/2500 [00:56<10:52,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 279/2500 [00:57<08:41,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 282/2500 [00:57<05:19,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 285/2500 [00:57<04:50,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 288/2500 [00:58<05:45,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 290/2500 [00:58<05:39,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 292/2500 [00:59<06:35,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 294/2500 [00:59<06:26,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 295/2500 [00:59<05:52,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 297/2500 [01:00<06:41,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 299/2500 [01:00<08:16,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 300/2500 [01:01<08:52,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 304/2500 [01:01<05:36,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 307/2500 [01:01<04:20,  8.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 309/2500 [01:02<05:51,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 311/2500 [01:02<05:43,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 313/2500 [01:02<05:23,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 314/2500 [01:03<10:39,  3.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 318/2500 [01:04<05:48,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 321/2500 [01:04<04:46,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 323/2500 [01:04<05:38,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 325/2500 [01:04<05:13,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 328/2500 [01:05<05:30,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 330/2500 [01:05<06:10,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 332/2500 [01:06<05:32,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 333/2500 [01:06<06:17,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 335/2500 [01:06<05:30,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 336/2500 [01:07<08:23,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 341/2500 [01:07<05:07,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 342/2500 [01:08<07:28,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 343/2500 [01:08<09:08,  3.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 348/2500 [01:09<05:34,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 350/2500 [01:09<05:19,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 352/2500 [01:09<05:49,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 353/2500 [01:10<06:11,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 356/2500 [01:10<05:36,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 359/2500 [01:10<03:57,  9.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 362/2500 [01:11<04:36,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 364/2500 [01:11<07:42,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 365/2500 [01:12<07:05,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 367/2500 [01:12<06:11,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 372/2500 [01:12<03:45,  9.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 374/2500 [01:13<04:39,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 378/2500 [01:13<04:41,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 382/2500 [01:14<05:29,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 384/2500 [01:15<07:37,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 385/2500 [01:15<06:42,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 388/2500 [01:15<06:28,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 391/2500 [01:16<06:14,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 392/2500 [01:16<05:40,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 395/2500 [01:16<05:00,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 397/2500 [01:17<04:14,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 398/2500 [01:17<04:44,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 399/2500 [01:17<06:52,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 400/2500 [01:17<08:13,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 403/2500 [01:18<06:14,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 404/2500 [01:18<06:45,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 406/2500 [01:18<06:27,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 408/2500 [01:19<07:00,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 412/2500 [01:20<07:07,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 413/2500 [01:20<09:48,  3.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 416/2500 [01:21<06:55,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 418/2500 [01:21<05:43,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 420/2500 [01:21<06:34,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 424/2500 [01:22<07:13,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 427/2500 [01:23<06:04,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 430/2500 [01:23<05:55,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 431/2500 [01:23<06:31,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 435/2500 [01:24<05:00,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 437/2500 [01:24<04:55,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 438/2500 [01:24<04:46,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 439/2500 [01:25<06:03,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 441/2500 [01:25<05:01,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 444/2500 [01:26<06:26,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 447/2500 [01:26<04:14,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 449/2500 [01:26<04:58,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 451/2500 [01:27<06:00,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 453/2500 [01:27<05:44,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 455/2500 [01:27<05:41,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 457/2500 [01:27<04:09,  8.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 458/2500 [01:28<05:16,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 459/2500 [01:28<08:03,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 463/2500 [01:28<04:33,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 465/2500 [01:29<06:14,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 467/2500 [01:29<05:28,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 468/2500 [01:30<06:21,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 470/2500 [01:30<05:27,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 471/2500 [01:30<07:48,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 474/2500 [01:31<06:06,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 476/2500 [01:31<05:18,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 477/2500 [01:31<07:35,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 478/2500 [01:32<09:04,  3.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 481/2500 [01:32<06:21,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 485/2500 [01:33<04:04,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 487/2500 [01:33<04:23,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 488/2500 [01:33<04:55,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 490/2500 [01:34<07:05,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 492/2500 [01:34<06:19,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 493/2500 [01:34<06:20,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 495/2500 [01:34<05:11,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 496/2500 [01:35<06:05,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 498/2500 [01:35<06:25,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 499/2500 [01:35<06:09,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 500/2500 [01:36<06:45,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 503/2500 [01:36<06:03,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 505/2500 [01:36<04:38,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 507/2500 [01:36<04:03,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 509/2500 [01:37<04:59,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 510/2500 [01:37<05:20,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 512/2500 [01:38<09:33,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 517/2500 [01:38<04:44,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 519/2500 [01:39<05:26,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 521/2500 [01:39<06:23,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 522/2500 [01:40<06:33,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 527/2500 [01:40<04:02,  8.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 529/2500 [01:40<04:44,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 531/2500 [01:41<06:31,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 534/2500 [01:42<06:16,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 536/2500 [01:42<06:07,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 539/2500 [01:42<04:42,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 542/2500 [01:43<04:00,  8.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 543/2500 [01:43<04:36,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 544/2500 [01:43<07:16,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 548/2500 [01:44<05:17,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 550/2500 [01:45<07:01,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 552/2500 [01:45<06:17,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 553/2500 [01:45<07:10,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 557/2500 [01:46<05:35,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 558/2500 [01:46<05:40,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 561/2500 [01:46<04:56,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 564/2500 [01:46<03:09, 10.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 566/2500 [01:47<02:57, 10.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 568/2500 [01:48<07:01,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 569/2500 [01:48<07:27,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 572/2500 [01:48<06:13,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 573/2500 [01:49<06:32,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 577/2500 [01:49<05:01,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 579/2500 [01:50<05:57,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 582/2500 [01:50<05:23,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 585/2500 [01:51<05:37,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 589/2500 [01:51<04:30,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 590/2500 [01:51<04:41,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 591/2500 [01:51<05:20,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 594/2500 [01:52<05:11,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 597/2500 [01:52<04:33,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 601/2500 [01:53<03:43,  8.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 602/2500 [01:53<03:37,  8.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 603/2500 [01:53<06:08,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 607/2500 [01:54<04:12,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 608/2500 [01:54<05:39,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 610/2500 [01:55<07:27,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 615/2500 [01:55<04:55,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 617/2500 [01:56<04:45,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 619/2500 [01:56<05:32,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 621/2500 [01:57<05:23,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 623/2500 [01:57<05:29,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 625/2500 [01:57<05:33,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 628/2500 [01:58<04:43,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 631/2500 [01:58<04:17,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 634/2500 [01:59<04:53,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 638/2500 [01:59<04:12,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 641/2500 [02:01<07:43,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 642/2500 [02:01<06:52,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 647/2500 [02:01<04:35,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 649/2500 [02:01<04:02,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 651/2500 [02:02<04:22,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 655/2500 [02:02<04:04,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 657/2500 [02:03<04:46,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 660/2500 [02:03<04:51,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 661/2500 [02:04<06:53,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 665/2500 [02:04<04:38,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 667/2500 [02:05<05:42,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 669/2500 [02:05<05:06,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 670/2500 [02:05<05:44,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 675/2500 [02:06<03:47,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 678/2500 [02:06<04:34,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 680/2500 [02:07<04:33,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 684/2500 [02:07<05:11,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 687/2500 [02:08<03:39,  8.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 689/2500 [02:08<04:26,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 693/2500 [02:09<04:22,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 695/2500 [02:09<04:17,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 696/2500 [02:09<06:22,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 699/2500 [02:10<05:53,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 700/2500 [02:10<06:14,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 704/2500 [02:11<04:05,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 707/2500 [02:11<03:32,  8.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 710/2500 [02:11<02:46, 10.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 712/2500 [02:12<04:04,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 713/2500 [02:12<05:45,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 715/2500 [02:12<05:51,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 717/2500 [02:13<05:06,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 718/2500 [02:13<04:43,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 720/2500 [02:14<06:45,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 725/2500 [02:14<03:33,  8.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 727/2500 [02:14<02:55, 10.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 731/2500 [02:15<03:41,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 733/2500 [02:15<04:21,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 735/2500 [02:16<04:56,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 738/2500 [02:16<05:36,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 740/2500 [02:17<05:05,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 742/2500 [02:17<03:41,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 744/2500 [02:17<04:22,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 746/2500 [02:17<04:33,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 748/2500 [02:18<03:30,  8.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 751/2500 [02:18<03:09,  9.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 753/2500 [02:19<07:04,  4.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 755/2500 [02:19<05:49,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 760/2500 [02:20<04:27,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 763/2500 [02:20<03:12,  9.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 767/2500 [02:21<03:56,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 769/2500 [02:21<04:42,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 770/2500 [02:21<05:56,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 777/2500 [02:22<03:31,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 779/2500 [02:22<03:30,  8.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 781/2500 [02:23<03:19,  8.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 783/2500 [02:23<03:24,  8.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 784/2500 [02:23<03:50,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 785/2500 [02:24<05:16,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 786/2500 [02:24<06:03,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 788/2500 [02:24<05:43,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 789/2500 [02:25<07:01,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 791/2500 [02:25<05:53,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 795/2500 [02:25<04:03,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 797/2500 [02:26<04:45,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 799/2500 [02:26<03:35,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 803/2500 [02:26<02:44, 10.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 805/2500 [02:27<03:25,  8.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 807/2500 [02:27<04:15,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 808/2500 [02:27<05:22,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 809/2500 [02:28<07:19,  3.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 811/2500 [02:28<06:28,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 812/2500 [02:29<06:45,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 816/2500 [02:29<04:11,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 819/2500 [02:29<03:35,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 821/2500 [02:30<03:47,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 823/2500 [02:30<04:01,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 826/2500 [02:30<02:39, 10.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 828/2500 [02:30<03:43,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 829/2500 [02:31<04:12,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 830/2500 [02:31<05:53,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 833/2500 [02:32<05:36,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 837/2500 [02:32<03:12,  8.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 841/2500 [02:33<02:58,  9.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 843/2500 [02:33<04:49,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 846/2500 [02:34<04:20,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 847/2500 [02:34<04:04,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 848/2500 [02:34<04:39,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 850/2500 [02:35<06:46,  4.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 853/2500 [02:35<05:04,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 856/2500 [02:36<04:30,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 858/2500 [02:36<03:51,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 861/2500 [02:36<04:09,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 864/2500 [02:37<03:46,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 865/2500 [02:37<04:25,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 868/2500 [02:38<05:08,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 870/2500 [02:38<06:51,  3.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 871/2500 [02:39<07:28,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 874/2500 [02:39<05:55,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 878/2500 [02:40<04:19,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 881/2500 [02:40<03:55,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 883/2500 [02:40<03:24,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 885/2500 [02:40<03:39,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 887/2500 [02:41<03:24,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 889/2500 [02:41<03:51,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 890/2500 [02:41<03:36,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 891/2500 [02:41<04:36,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 893/2500 [02:42<06:23,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 895/2500 [02:42<05:27,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 897/2500 [02:43<05:26,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 898/2500 [02:43<04:55,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 900/2500 [02:43<05:41,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 902/2500 [02:44<05:50,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 904/2500 [02:44<05:22,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 907/2500 [02:45<03:57,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 908/2500 [02:45<04:48,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 909/2500 [02:45<05:14,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 913/2500 [02:46<04:04,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 915/2500 [02:46<04:42,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 916/2500 [02:46<05:55,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 917/2500 [02:47<08:18,  3.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 920/2500 [02:47<05:17,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 923/2500 [02:48<04:13,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 924/2500 [02:48<05:19,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 928/2500 [02:48<03:33,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 930/2500 [02:49<03:12,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 932/2500 [02:49<04:27,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 935/2500 [02:50<04:15,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 937/2500 [02:50<05:30,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 939/2500 [02:51<05:10,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 943/2500 [02:51<03:29,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 945/2500 [02:51<03:13,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 947/2500 [02:52<04:07,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 951/2500 [02:52<03:47,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 952/2500 [02:53<04:22,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 953/2500 [02:53<04:33,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 956/2500 [02:54<06:00,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 958/2500 [02:54<04:53,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 960/2500 [02:54<04:52,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 965/2500 [02:55<03:18,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 967/2500 [02:55<03:52,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 968/2500 [02:55<04:19,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 969/2500 [02:56<04:47,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 973/2500 [02:56<03:49,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 974/2500 [02:57<05:32,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 975/2500 [02:57<06:26,  3.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 979/2500 [02:57<03:49,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 983/2500 [02:58<02:55,  8.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 985/2500 [02:58<03:09,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 987/2500 [02:58<02:58,  8.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 989/2500 [02:58<03:18,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 990/2500 [02:59<04:03,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 991/2500 [02:59<04:21,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 993/2500 [03:00<05:05,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 996/2500 [03:00<03:54,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 998/2500 [03:00<04:33,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1002/2500 [03:01<03:13,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1004/2500 [03:01<04:39,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1005/2500 [03:02<06:39,  3.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1007/2500 [03:02<05:57,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1009/2500 [03:03<05:21,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1015/2500 [03:03<03:13,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1017/2500 [03:03<03:13,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1019/2500 [03:04<04:11,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1021/2500 [03:04<03:49,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1022/2500 [03:05<05:00,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1023/2500 [03:05<05:03,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1025/2500 [03:05<05:56,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1027/2500 [03:06<04:58,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1031/2500 [03:06<03:47,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1033/2500 [03:06<03:30,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1035/2500 [03:07<02:50,  8.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1037/2500 [03:07<02:51,  8.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1039/2500 [03:07<03:08,  7.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1040/2500 [03:08<05:02,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1041/2500 [03:08<05:14,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1045/2500 [03:09<04:57,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1047/2500 [03:09<04:23,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1050/2500 [03:09<03:03,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1052/2500 [03:10<03:14,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1054/2500 [03:10<03:26,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1057/2500 [03:10<02:28,  9.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1059/2500 [03:11<04:28,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1063/2500 [03:12<03:51,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1066/2500 [03:12<04:43,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1069/2500 [03:13<03:27,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1072/2500 [03:13<03:22,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1073/2500 [03:13<03:36,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1076/2500 [03:14<03:26,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1078/2500 [03:14<04:29,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1080/2500 [03:15<05:55,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1083/2500 [03:15<04:12,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1085/2500 [03:16<03:56,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1088/2500 [03:16<03:55,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▎     | 1090/2500 [03:17<05:24,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1097/2500 [03:17<02:41,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1098/2500 [03:18<03:19,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1099/2500 [03:18<03:36,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1103/2500 [03:19<03:59,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1107/2500 [03:19<02:31,  9.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1109/2500 [03:20<03:46,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1112/2500 [03:20<03:47,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1114/2500 [03:21<04:43,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1117/2500 [03:21<04:48,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1118/2500 [03:22<06:03,  3.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1120/2500 [03:22<05:20,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1125/2500 [03:23<02:53,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1127/2500 [03:23<02:51,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1131/2500 [03:23<02:33,  8.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1134/2500 [03:24<02:52,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1135/2500 [03:24<03:18,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1136/2500 [03:24<04:59,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1138/2500 [03:25<06:36,  3.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1139/2500 [03:25<05:39,  4.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1142/2500 [03:26<04:02,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1145/2500 [03:26<02:27,  9.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1147/2500 [03:26<02:16,  9.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1149/2500 [03:26<02:55,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1151/2500 [03:27<02:55,  7.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1152/2500 [03:27<03:22,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1154/2500 [03:28<04:32,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1155/2500 [03:28<05:04,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 1158/2500 [03:28<04:12,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1160/2500 [03:28<03:02,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1162/2500 [03:29<02:27,  9.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1167/2500 [03:30<03:42,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1169/2500 [03:30<03:28,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1171/2500 [03:30<03:26,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1174/2500 [03:31<03:56,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1177/2500 [03:31<02:39,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1179/2500 [03:31<02:47,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1181/2500 [03:32<03:18,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1182/2500 [03:32<03:28,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1184/2500 [03:33<03:54,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1185/2500 [03:33<05:01,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1188/2500 [03:34<04:58,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1190/2500 [03:34<03:53,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1193/2500 [03:34<03:05,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1196/2500 [03:34<02:20,  9.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1198/2500 [03:35<03:15,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1203/2500 [03:35<02:37,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1205/2500 [03:36<03:00,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1206/2500 [03:37<04:55,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1208/2500 [03:37<04:58,  4.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1210/2500 [03:37<04:56,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1211/2500 [03:38<04:18,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▊     | 1214/2500 [03:38<03:42,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▊     | 1217/2500 [03:39<03:54,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1219/2500 [03:39<03:18,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1221/2500 [03:39<03:41,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1223/2500 [03:40<03:48,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1224/2500 [03:40<04:04,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1226/2500 [03:40<03:25,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1227/2500 [03:41<04:36,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1230/2500 [03:41<03:32,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1232/2500 [03:41<03:03,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1235/2500 [03:41<02:47,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1237/2500 [03:42<02:36,  8.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1238/2500 [03:42<03:17,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1239/2500 [03:42<04:31,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1242/2500 [03:43<04:21,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1244/2500 [03:43<03:08,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1246/2500 [03:44<03:36,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1248/2500 [03:44<03:14,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1249/2500 [03:44<03:45,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1251/2500 [03:44<03:28,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1253/2500 [03:45<03:08,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1254/2500 [03:45<03:39,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1255/2500 [03:45<03:49,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1256/2500 [03:45<04:06,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1259/2500 [03:46<02:51,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1260/2500 [03:46<03:18,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1264/2500 [03:47<03:05,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1267/2500 [03:47<02:05,  9.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1269/2500 [03:47<03:06,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1271/2500 [03:48<03:54,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1275/2500 [03:48<02:49,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1279/2500 [03:49<02:31,  8.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1282/2500 [03:49<03:37,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1284/2500 [03:50<03:36,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1289/2500 [03:50<02:44,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1291/2500 [03:51<02:50,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1294/2500 [03:51<02:44,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1297/2500 [03:53<04:56,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1298/2500 [03:53<04:34,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1302/2500 [03:53<03:16,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1306/2500 [03:54<02:40,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1308/2500 [03:54<03:20,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1311/2500 [03:54<02:33,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1312/2500 [03:55<02:56,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1316/2500 [03:55<02:29,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1318/2500 [03:56<03:49,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1320/2500 [03:56<03:46,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1323/2500 [03:57<03:18,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1325/2500 [03:57<03:15,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1327/2500 [03:58<05:09,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1330/2500 [03:58<03:26,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1332/2500 [03:58<03:15,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1336/2500 [03:59<02:24,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1338/2500 [03:59<01:55, 10.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1340/2500 [03:59<02:38,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1342/2500 [04:00<03:32,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1343/2500 [04:00<03:55,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1345/2500 [04:01<04:05,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1347/2500 [04:01<03:22,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1349/2500 [04:01<03:45,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1350/2500 [04:01<03:40,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1353/2500 [04:02<03:06,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1358/2500 [04:02<01:47, 10.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1362/2500 [04:03<02:25,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1364/2500 [04:04<04:11,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1365/2500 [04:04<04:57,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1366/2500 [04:04<04:59,  3.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1373/2500 [04:05<02:09,  8.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1376/2500 [04:05<01:42, 10.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1378/2500 [04:06<03:14,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1380/2500 [04:06<03:36,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1383/2500 [04:07<03:06,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1384/2500 [04:07<03:41,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1385/2500 [04:07<04:02,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1386/2500 [04:07<03:58,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1388/2500 [04:08<03:47,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1392/2500 [04:08<02:18,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1394/2500 [04:09<02:44,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1395/2500 [04:09<03:11,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1397/2500 [04:09<03:10,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1400/2500 [04:10<03:10,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1402/2500 [04:10<03:12,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1403/2500 [04:10<03:53,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1406/2500 [04:11<03:23,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1409/2500 [04:11<02:35,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1411/2500 [04:11<02:30,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1414/2500 [04:12<02:35,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1415/2500 [04:12<02:31,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1418/2500 [04:12<02:19,  7.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1419/2500 [04:13<03:31,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1421/2500 [04:13<03:50,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1424/2500 [04:14<02:39,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1427/2500 [04:14<02:17,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1429/2500 [04:14<02:27,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1432/2500 [04:15<02:20,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1433/2500 [04:15<02:23,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1434/2500 [04:15<03:05,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1436/2500 [04:16<03:56,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1438/2500 [04:16<02:54,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1440/2500 [04:16<01:59,  8.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1442/2500 [04:16<02:08,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1445/2500 [04:17<03:08,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1446/2500 [04:18<04:16,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1449/2500 [04:18<02:50,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1454/2500 [04:19<03:35,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1455/2500 [04:19<03:36,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1458/2500 [04:20<02:40,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1461/2500 [04:20<02:15,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1462/2500 [04:20<02:36,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▊    | 1465/2500 [04:20<02:18,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▊    | 1467/2500 [04:21<02:25,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1469/2500 [04:22<04:10,  4.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1470/2500 [04:22<05:52,  2.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1474/2500 [04:23<03:16,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1475/2500 [04:23<03:57,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1477/2500 [04:23<03:16,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1479/2500 [04:24<02:59,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1483/2500 [04:24<02:11,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1487/2500 [04:24<01:36, 10.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1489/2500 [04:25<01:57,  8.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1492/2500 [04:26<03:22,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1494/2500 [04:26<04:12,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1496/2500 [04:27<03:46,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1500/2500 [04:27<01:49,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1504/2500 [04:27<01:32, 10.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1506/2500 [04:28<03:05,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1508/2500 [04:28<03:31,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1512/2500 [04:29<02:49,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1514/2500 [04:30<03:17,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1515/2500 [04:30<03:58,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1520/2500 [04:30<02:06,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1522/2500 [04:31<02:14,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1525/2500 [04:31<02:00,  8.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1527/2500 [04:32<02:38,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1530/2500 [04:32<02:22,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 1533/2500 [04:33<03:40,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 1536/2500 [04:33<02:42,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1537/2500 [04:34<02:36,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1539/2500 [04:34<03:17,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1542/2500 [04:34<02:22,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1543/2500 [04:35<02:32,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1545/2500 [04:35<02:40,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1546/2500 [04:35<02:56,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1547/2500 [04:36<03:47,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1550/2500 [04:36<03:04,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1551/2500 [04:36<02:47,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1554/2500 [04:37<02:04,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1558/2500 [04:37<02:02,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1559/2500 [04:37<02:00,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1561/2500 [04:38<02:26,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1564/2500 [04:39<03:20,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1565/2500 [04:39<03:28,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1569/2500 [04:39<02:29,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1571/2500 [04:39<02:08,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1574/2500 [04:40<02:00,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1576/2500 [04:41<03:20,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1578/2500 [04:41<02:26,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1581/2500 [04:41<01:55,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1584/2500 [04:41<01:43,  8.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1586/2500 [04:42<02:15,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1587/2500 [04:42<02:27,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1591/2500 [04:43<02:26,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1592/2500 [04:43<02:17,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1595/2500 [04:43<02:16,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1597/2500 [04:44<02:30,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1599/2500 [04:44<02:38,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1601/2500 [04:44<02:35,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1603/2500 [04:45<02:12,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1606/2500 [04:45<02:16,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1607/2500 [04:45<02:54,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1610/2500 [04:46<02:13,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1611/2500 [04:46<03:04,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1613/2500 [04:47<04:07,  3.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1615/2500 [04:47<03:29,  4.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1620/2500 [04:48<02:12,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1621/2500 [04:48<02:04,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1624/2500 [04:48<01:59,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1626/2500 [04:49<02:22,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1627/2500 [04:49<02:51,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1630/2500 [04:49<02:16,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1631/2500 [04:50<03:01,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1633/2500 [04:50<02:33,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1635/2500 [04:51<02:49,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1638/2500 [04:51<02:04,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1642/2500 [04:51<01:46,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1644/2500 [04:52<02:08,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1646/2500 [04:52<01:44,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1648/2500 [04:52<02:20,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1650/2500 [04:53<03:01,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1652/2500 [04:53<02:49,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1653/2500 [04:54<03:03,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1657/2500 [04:54<02:12,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1660/2500 [04:54<01:39,  8.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1662/2500 [04:55<01:43,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1664/2500 [04:55<02:05,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1667/2500 [04:55<01:59,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1673/2500 [04:56<02:00,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1675/2500 [04:57<03:09,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1677/2500 [04:58<02:38,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1680/2500 [04:58<02:30,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1682/2500 [04:58<02:23,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1684/2500 [04:59<01:46,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1686/2500 [04:59<02:11,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1688/2500 [04:59<02:05,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1690/2500 [05:00<02:23,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1693/2500 [05:00<01:39,  8.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1694/2500 [05:00<01:47,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1695/2500 [05:00<02:24,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1697/2500 [05:01<02:28,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1699/2500 [05:01<02:16,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1700/2500 [05:01<02:42,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1703/2500 [05:02<02:01,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1704/2500 [05:02<01:52,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1707/2500 [05:02<01:42,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1709/2500 [05:03<01:33,  8.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1710/2500 [05:03<01:47,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 1713/2500 [05:03<01:51,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 1714/2500 [05:04<04:00,  3.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 1716/2500 [05:04<03:19,  3.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 1717/2500 [05:05<03:08,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1720/2500 [05:05<02:04,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1722/2500 [05:05<02:16,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1724/2500 [05:06<01:57,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1726/2500 [05:06<01:49,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1727/2500 [05:06<02:22,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 1730/2500 [05:07<02:42,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 1732/2500 [05:07<02:18,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1734/2500 [05:07<01:40,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1735/2500 [05:08<01:59,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1736/2500 [05:08<02:35,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 1740/2500 [05:08<01:53,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 1745/2500 [05:09<01:38,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 1747/2500 [05:09<01:53,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 1749/2500 [05:10<02:08,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1750/2500 [05:10<02:20,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1751/2500 [05:10<02:47,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1753/2500 [05:11<02:20,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1755/2500 [05:11<02:12,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1757/2500 [05:11<02:13,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1758/2500 [05:12<02:35,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1761/2500 [05:12<02:06,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1765/2500 [05:13<01:50,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1768/2500 [05:13<02:05,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1769/2500 [05:13<02:06,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1771/2500 [05:14<02:24,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1775/2500 [05:14<02:02,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1778/2500 [05:15<01:29,  8.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1779/2500 [05:15<02:04,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 1782/2500 [05:15<01:43,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 1783/2500 [05:16<01:41,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 1784/2500 [05:16<02:01,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1788/2500 [05:16<01:32,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1790/2500 [05:16<01:21,  8.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1791/2500 [05:17<02:06,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1795/2500 [05:17<01:46,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1797/2500 [05:18<02:24,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1801/2500 [05:19<01:32,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1804/2500 [05:19<01:29,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1807/2500 [05:19<01:32,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1811/2500 [05:20<01:32,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1813/2500 [05:21<02:30,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1815/2500 [05:21<02:24,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1818/2500 [05:22<01:56,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1822/2500 [05:22<01:09,  9.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1824/2500 [05:22<01:10,  9.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1826/2500 [05:22<01:36,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1828/2500 [05:23<01:33,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1830/2500 [05:23<01:53,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1831/2500 [05:24<02:16,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1834/2500 [05:24<01:53,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▎  | 1838/2500 [05:25<01:49,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 1841/2500 [05:25<01:16,  8.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 1843/2500 [05:25<01:14,  8.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1845/2500 [05:25<01:29,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1847/2500 [05:26<01:32,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1848/2500 [05:26<01:52,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1851/2500 [05:27<01:45,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1854/2500 [05:27<01:47,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1855/2500 [05:27<01:51,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1857/2500 [05:28<02:25,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1861/2500 [05:28<01:31,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1862/2500 [05:29<01:33,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 1865/2500 [05:29<01:19,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 1868/2500 [05:29<01:04,  9.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 1870/2500 [05:29<01:07,  9.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 1871/2500 [05:30<01:19,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 1872/2500 [05:30<01:37,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 1874/2500 [05:31<02:19,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 1876/2500 [05:31<02:12,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1878/2500 [05:31<01:51,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1879/2500 [05:31<01:47,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1880/2500 [05:32<02:08,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 1884/2500 [05:32<01:26,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1887/2500 [05:33<01:30,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1891/2500 [05:33<01:22,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1893/2500 [05:34<01:33,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1897/2500 [05:34<01:26,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1899/2500 [05:35<01:57,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1901/2500 [05:35<01:47,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1903/2500 [05:35<01:40,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1906/2500 [05:36<00:56, 10.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 1908/2500 [05:36<01:32,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 1910/2500 [05:36<01:35,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▋  | 1912/2500 [05:37<01:42,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1914/2500 [05:37<01:45,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1916/2500 [05:37<01:32,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1918/2500 [05:38<01:28,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1920/2500 [05:38<01:45,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1925/2500 [05:39<00:59,  9.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1927/2500 [05:39<01:24,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1931/2500 [05:39<01:05,  8.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1935/2500 [05:40<01:30,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1937/2500 [05:41<01:22,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1939/2500 [05:41<01:24,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1943/2500 [05:42<01:32,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1945/2500 [05:42<01:34,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1949/2500 [05:43<01:11,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1951/2500 [05:43<00:59,  9.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1953/2500 [05:44<02:13,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1955/2500 [05:44<01:56,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1957/2500 [05:44<01:36,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1958/2500 [05:45<02:23,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1962/2500 [05:45<01:25,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 1964/2500 [05:46<01:20,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 1966/2500 [05:46<01:05,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1969/2500 [05:46<01:18,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1971/2500 [05:47<01:08,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1972/2500 [05:47<01:30,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1973/2500 [05:48<02:22,  3.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1976/2500 [05:48<01:31,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1977/2500 [05:48<01:54,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1979/2500 [05:49<01:38,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1983/2500 [05:49<01:08,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 1988/2500 [05:49<00:53,  9.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 1990/2500 [05:50<00:54,  9.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 1992/2500 [05:50<01:18,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 1993/2500 [05:51<01:43,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 1994/2500 [05:51<01:51,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 1998/2500 [05:52<01:34,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2000/2500 [05:52<01:11,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2001/2500 [05:52<01:25,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2003/2500 [05:52<01:25,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2006/2500 [05:52<00:55,  8.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2008/2500 [05:53<00:55,  8.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2010/2500 [05:53<01:12,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2011/2500 [05:54<01:48,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2013/2500 [05:54<01:44,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2016/2500 [05:54<01:11,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2018/2500 [05:55<01:22,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2020/2500 [05:55<01:26,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2024/2500 [05:56<01:00,  7.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2026/2500 [05:56<00:53,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2028/2500 [05:56<01:01,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2030/2500 [05:56<00:58,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2031/2500 [05:57<01:30,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2033/2500 [05:57<01:47,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2035/2500 [05:58<01:39,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2037/2500 [05:58<01:15,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2041/2500 [05:58<00:57,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2044/2500 [05:59<00:52,  8.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2046/2500 [05:59<00:43, 10.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2048/2500 [05:59<00:45,  9.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2050/2500 [05:59<00:54,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2051/2500 [06:00<01:23,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2052/2500 [06:00<01:41,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2054/2500 [06:01<01:44,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2057/2500 [06:01<01:06,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2058/2500 [06:01<01:04,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2061/2500 [06:02<01:02,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2065/2500 [06:02<01:03,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2067/2500 [06:02<00:59,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2068/2500 [06:03<00:57,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2069/2500 [06:03<01:26,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2073/2500 [06:04<01:04,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2075/2500 [06:04<01:03,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2078/2500 [06:04<00:46,  9.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2080/2500 [06:04<00:52,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2082/2500 [06:05<01:07,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2084/2500 [06:05<00:57,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2085/2500 [06:05<01:12,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2089/2500 [06:06<01:23,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2091/2500 [06:07<01:10,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2092/2500 [06:07<01:08,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2093/2500 [06:07<01:14,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2097/2500 [06:07<00:56,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2099/2500 [06:08<00:45,  8.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2101/2500 [06:09<01:52,  3.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2106/2500 [06:09<01:08,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2107/2500 [06:10<01:19,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2110/2500 [06:10<01:04,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2113/2500 [06:10<00:48,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2115/2500 [06:10<00:43,  8.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2117/2500 [06:11<00:53,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2121/2500 [06:12<01:03,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2123/2500 [06:12<01:03,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2129/2500 [06:13<00:48,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2131/2500 [06:13<01:01,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2133/2500 [06:14<00:54,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2135/2500 [06:14<00:55,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2136/2500 [06:14<00:53,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2137/2500 [06:14<01:07,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2139/2500 [06:15<01:09,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2141/2500 [06:15<01:04,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2144/2500 [06:15<00:47,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2146/2500 [06:16<00:48,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2148/2500 [06:16<00:46,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2149/2500 [06:16<00:51,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2151/2500 [06:17<01:03,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2153/2500 [06:17<01:02,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2155/2500 [06:17<00:47,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2156/2500 [06:17<00:58,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▋ | 2159/2500 [06:18<00:52,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2160/2500 [06:18<01:03,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2163/2500 [06:18<00:48,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2165/2500 [06:19<00:52,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2169/2500 [06:19<00:40,  8.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2171/2500 [06:20<00:52,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2172/2500 [06:20<00:49,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2176/2500 [06:21<00:47,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2177/2500 [06:21<01:03,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2181/2500 [06:22<00:52,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2182/2500 [06:22<00:50,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2185/2500 [06:22<00:44,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2186/2500 [06:22<00:54,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2190/2500 [06:23<00:50,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2191/2500 [06:23<00:48,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2194/2500 [06:23<00:44,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2195/2500 [06:24<00:45,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2196/2500 [06:24<01:02,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2199/2500 [06:24<00:46,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2201/2500 [06:25<00:42,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2205/2500 [06:25<00:44,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2207/2500 [06:26<00:41,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2208/2500 [06:26<00:53,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2210/2500 [06:26<00:55,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2212/2500 [06:26<00:38,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 2215/2500 [06:27<00:45,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 2217/2500 [06:27<00:45,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2219/2500 [06:28<00:55,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2222/2500 [06:28<00:45,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2223/2500 [06:29<00:54,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2225/2500 [06:29<00:47,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2227/2500 [06:29<00:45,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2228/2500 [06:29<00:52,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2232/2500 [06:30<00:33,  7.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2234/2500 [06:30<00:32,  8.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2235/2500 [06:30<00:36,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2236/2500 [06:31<00:56,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2237/2500 [06:31<01:03,  4.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2242/2500 [06:32<00:45,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2244/2500 [06:32<00:36,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2248/2500 [06:32<00:29,  8.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2250/2500 [06:33<00:36,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2254/2500 [06:33<00:33,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2256/2500 [06:34<00:46,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2259/2500 [06:35<01:06,  3.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2264/2500 [06:36<00:37,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2266/2500 [06:36<00:31,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2268/2500 [06:36<00:33,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2270/2500 [06:36<00:34,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2273/2500 [06:37<00:32,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2276/2500 [06:38<00:47,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2280/2500 [06:38<00:33,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2283/2500 [06:39<00:39,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2286/2500 [06:39<00:28,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2288/2500 [06:40<00:36,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2291/2500 [06:40<00:26,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2292/2500 [06:40<00:32,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2294/2500 [06:41<00:47,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2296/2500 [06:41<00:39,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2300/2500 [06:42<00:28,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2301/2500 [06:42<00:36,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2302/2500 [06:42<00:37,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2304/2500 [06:43<00:34,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2309/2500 [06:43<00:16, 11.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2311/2500 [06:43<00:18, 10.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2313/2500 [06:44<00:39,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2315/2500 [06:44<00:33,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2316/2500 [06:45<00:40,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2319/2500 [06:45<00:32,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2320/2500 [06:45<00:35,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2324/2500 [06:46<00:23,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2328/2500 [06:46<00:17,  9.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2330/2500 [06:47<00:24,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2331/2500 [06:47<00:26,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2332/2500 [06:47<00:27,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2333/2500 [06:47<00:34,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2334/2500 [06:48<00:41,  4.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2335/2500 [06:48<00:39,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2336/2500 [06:48<00:37,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▎| 2341/2500 [06:49<00:21,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2345/2500 [06:49<00:14, 10.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2347/2500 [06:50<00:25,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2348/2500 [06:50<00:29,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2350/2500 [06:50<00:28,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2353/2500 [06:51<00:24,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2356/2500 [06:51<00:19,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2357/2500 [06:51<00:20,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2360/2500 [06:52<00:24,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2363/2500 [06:52<00:16,  8.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2366/2500 [06:52<00:16,  8.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2370/2500 [06:54<00:23,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2374/2500 [06:54<00:15,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2376/2500 [06:54<00:13,  9.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2378/2500 [06:54<00:17,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2381/2500 [06:55<00:24,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2382/2500 [06:56<00:26,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2387/2500 [06:56<00:16,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2389/2500 [06:57<00:19,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2390/2500 [06:57<00:19,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2391/2500 [06:57<00:23,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2392/2500 [06:57<00:23,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2395/2500 [06:58<00:16,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2396/2500 [06:58<00:19,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2399/2500 [06:58<00:14,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2401/2500 [06:59<00:12,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2406/2500 [06:59<00:08, 10.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2408/2500 [07:00<00:19,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2409/2500 [07:01<00:25,  3.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2411/2500 [07:01<00:20,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2414/2500 [07:01<00:14,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2418/2500 [07:01<00:10,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2420/2500 [07:02<00:10,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2425/2500 [07:02<00:06, 10.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2428/2500 [07:03<00:10,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2429/2500 [07:03<00:16,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2430/2500 [07:04<00:17,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2432/2500 [07:04<00:15,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2436/2500 [07:05<00:09,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2438/2500 [07:05<00:07,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2440/2500 [07:05<00:09,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2442/2500 [07:05<00:08,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2445/2500 [07:06<00:08,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2448/2500 [07:07<00:09,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2450/2500 [07:07<00:11,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2455/2500 [07:08<00:05,  8.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2459/2500 [07:08<00:05,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2461/2500 [07:09<00:05,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2462/2500 [07:09<00:05,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2464/2500 [07:09<00:06,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▊| 2466/2500 [07:10<00:06,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2468/2500 [07:10<00:04,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2470/2500 [07:10<00:05,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2471/2500 [07:10<00:04,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2473/2500 [07:11<00:05,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2478/2500 [07:11<00:03,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2483/2500 [07:12<00:01, 10.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2485/2500 [07:13<00:02,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2487/2500 [07:14<00:03,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2491/2500 [07:14<00:01,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2494/2500 [07:14<00:00,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2498/2500 [07:15<00:00,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 2500/2500 [07:21<00:00,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-27 17:17:56.565\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m4\u001b[0m - \u001b[1mEvaluation metrics (before optimization): {'f1': 0.0004, 'em': 0.0004, 'acc': 0.2108}\u001b[0m\n",
      "\u001b[32m2025-12-27 17:17:56.565\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m6\u001b[0m - \u001b[1mOptimizing workflow...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/20 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:17:56.569\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:18:19.757\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:19:41.884\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:19:47.413\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:19:47.413\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 1 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:01<02:04,  1.58s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   2%|▎         | 2/80 [00:02<01:29,  1.15s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 3/80 [00:02<00:57,  1.34it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▋         | 5/80 [00:02<00:30,  2.49it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 6/80 [00:03<00:25,  2.91it/s]\u001b[A\n",
      "Evaluating workflow:  14%|█▍        | 11/80 [00:03<00:09,  7.48it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  16%|█▋        | 13/80 [00:03<00:08,  7.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▉        | 15/80 [00:03<00:09,  6.95it/s]\u001b[A\n",
      "Evaluating workflow:  24%|██▍       | 19/80 [00:04<00:05, 10.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  26%|██▋       | 21/80 [00:04<00:08,  7.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▉       | 23/80 [00:05<00:13,  4.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  31%|███▏      | 25/80 [00:06<00:12,  4.40it/s]\u001b[A\n",
      "Evaluating workflow:  34%|███▍      | 27/80 [00:06<00:09,  5.38it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 30/80 [00:06<00:06,  7.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  40%|████      | 32/80 [00:06<00:06,  6.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▎     | 34/80 [00:07<00:06,  6.86it/s]\u001b[A\n",
      "Evaluating workflow:  45%|████▌     | 36/80 [00:07<00:05,  7.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 38/80 [00:07<00:06,  6.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████▏    | 41/80 [00:07<00:04,  8.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 43/80 [00:08<00:08,  4.56it/s]\u001b[A\n",
      "Evaluating workflow:  55%|█████▌    | 44/80 [00:08<00:07,  4.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▋    | 45/80 [00:09<00:06,  5.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  57%|█████▊    | 46/80 [00:09<00:06,  4.92it/s]\u001b[A\n",
      "Evaluating workflow:  59%|█████▉    | 47/80 [00:09<00:06,  4.98it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▎   | 50/80 [00:09<00:03,  7.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▌   | 52/80 [00:10<00:04,  6.07it/s]\u001b[A\n",
      "Evaluating workflow:  68%|██████▊   | 54/80 [00:10<00:03,  7.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  70%|███████   | 56/80 [00:10<00:02,  8.21it/s]\u001b[A\n",
      "Evaluating workflow:  72%|███████▎  | 58/80 [00:10<00:02,  9.27it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▌  | 60/80 [00:10<00:02,  8.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 62/80 [00:11<00:03,  5.09it/s]\u001b[A\n",
      "Evaluating workflow:  80%|████████  | 64/80 [00:11<00:02,  6.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▎ | 66/80 [00:12<00:02,  5.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  85%|████████▌ | 68/80 [00:12<00:02,  5.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▋ | 69/80 [00:12<00:02,  4.82it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▉ | 71/80 [00:13<00:01,  5.57it/s]\u001b[A\n",
      "Evaluating workflow:  90%|█████████ | 72/80 [00:13<00:01,  5.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 73/80 [00:13<00:01,  5.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 75/80 [00:13<00:00,  5.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 76/80 [00:14<00:00,  4.58it/s]\u001b[A\n",
      "Evaluating workflow:  96%|█████████▋| 77/80 [00:14<00:00,  4.85it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 78/80 [00:14<00:00,  5.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 79/80 [00:15<00:00,  3.10it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 80/80 [00:16<00:00,  4.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2025-12-27 17:20:03.934\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 1 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.2625}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      "  5%|▌         | 1/20 [02:07<40:19, 127.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:20:03.935\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:20:18.234\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:21:48.741\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:21:52.828\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:21:52.828\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 2 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:02<03:39,  2.78s/it]\u001b[A\n",
      "Evaluating workflow:   2%|▎         | 2/80 [00:02<01:34,  1.21s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▋         | 5/80 [00:02<00:27,  2.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   9%|▉         | 7/80 [00:03<00:20,  3.63it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  14%|█▍        | 11/80 [00:03<00:10,  6.32it/s]\u001b[A\n",
      "Evaluating workflow:  19%|█▉        | 15/80 [00:03<00:06,  9.84it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  22%|██▎       | 18/80 [00:03<00:06, 10.14it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▌       | 20/80 [00:04<00:10,  5.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 22/80 [00:05<00:13,  4.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▉       | 23/80 [00:06<00:17,  3.21it/s]\u001b[A\n",
      "Evaluating workflow:  34%|███▍      | 27/80 [00:06<00:09,  5.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  36%|███▋      | 29/80 [00:06<00:09,  5.43it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████▏     | 33/80 [00:07<00:07,  6.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 38/80 [00:07<00:04,  9.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  50%|█████     | 40/80 [00:08<00:06,  6.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▎    | 42/80 [00:09<00:08,  4.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▌    | 44/80 [00:09<00:07,  4.57it/s]\u001b[A\n",
      "Evaluating workflow:  60%|██████    | 48/80 [00:09<00:04,  6.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  64%|██████▍   | 51/80 [00:10<00:04,  6.86it/s]\u001b[A\n",
      "Evaluating workflow:  66%|██████▋   | 53/80 [00:10<00:03,  7.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  70%|███████   | 56/80 [00:10<00:02,  9.05it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▎  | 58/80 [00:11<00:03,  5.81it/s]\u001b[A\n",
      "Evaluating workflow:  74%|███████▍  | 59/80 [00:11<00:03,  5.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▌  | 60/80 [00:11<00:04,  4.27it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 62/80 [00:12<00:03,  5.19it/s]\u001b[A\n",
      "Evaluating workflow:  80%|████████  | 64/80 [00:12<00:02,  6.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▎ | 66/80 [00:12<00:02,  5.50it/s]\u001b[A\n",
      "Evaluating workflow:  86%|████████▋ | 69/80 [00:12<00:01,  7.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▉ | 71/80 [00:13<00:01,  7.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 73/80 [00:13<00:01,  5.95it/s]\u001b[A\n",
      "Evaluating workflow:  92%|█████████▎| 74/80 [00:13<00:00,  6.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 75/80 [00:14<00:00,  6.69it/s]\u001b[A\n",
      "Evaluating workflow:  95%|█████████▌| 76/80 [00:14<00:00,  6.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▋| 77/80 [00:14<00:00,  5.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 78/80 [00:14<00:00,  4.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 79/80 [00:15<00:00,  4.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 80/80 [00:15<00:00,  5.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-27 17:22:08.577\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 2 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.325}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 10%|█         | 2/20 [04:12<37:43, 125.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:22:08.578\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:22:17.661\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:23:24.223\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:23:28.436\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:23:28.436\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 3 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:01<01:59,  1.51s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   2%|▎         | 2/80 [00:02<01:25,  1.10s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   5%|▌         | 4/80 [00:02<00:42,  1.78it/s]\u001b[A\n",
      "Evaluating workflow:  10%|█         | 8/80 [00:02<00:15,  4.58it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  14%|█▍        | 11/80 [00:03<00:10,  6.38it/s]\u001b[A\n",
      "Evaluating workflow:  16%|█▋        | 13/80 [00:03<00:08,  7.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▉        | 15/80 [00:03<00:09,  7.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██▏       | 17/80 [00:03<00:08,  7.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  24%|██▍       | 19/80 [00:04<00:09,  6.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▌       | 20/80 [00:04<00:10,  5.84it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 22/80 [00:04<00:10,  5.46it/s]\u001b[A\n",
      "Evaluating workflow:  29%|██▉       | 23/80 [00:05<00:09,  5.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 24/80 [00:05<00:10,  5.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  32%|███▎      | 26/80 [00:05<00:08,  6.27it/s]\u001b[A\n",
      "Evaluating workflow:  34%|███▍      | 27/80 [00:05<00:08,  6.17it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▌      | 28/80 [00:05<00:09,  5.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  36%|███▋      | 29/80 [00:06<00:11,  4.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 30/80 [00:06<00:13,  3.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▎     | 34/80 [00:07<00:07,  6.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 38/80 [00:07<00:05,  8.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▉     | 39/80 [00:07<00:05,  7.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  50%|█████     | 40/80 [00:08<00:08,  4.62it/s]\u001b[A\n",
      "Evaluating workflow:  52%|█████▎    | 42/80 [00:08<00:06,  6.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 43/80 [00:08<00:06,  5.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▌    | 44/80 [00:08<00:06,  5.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 48/80 [00:09<00:05,  6.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▎   | 50/80 [00:09<00:04,  6.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  64%|██████▍   | 51/80 [00:09<00:04,  6.20it/s]\u001b[A\n",
      "Evaluating workflow:  65%|██████▌   | 52/80 [00:09<00:04,  6.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▋   | 53/80 [00:10<00:06,  4.21it/s]\u001b[A\n",
      "Evaluating workflow:  69%|██████▉   | 55/80 [00:10<00:04,  5.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  70%|███████   | 56/80 [00:10<00:03,  6.05it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▌  | 60/80 [00:10<00:02,  8.42it/s]\u001b[A\n",
      "Evaluating workflow:  78%|███████▊  | 62/80 [00:11<00:01,  9.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 64/80 [00:11<00:02,  5.37it/s]\u001b[A\n",
      "Evaluating workflow:  81%|████████▏ | 65/80 [00:12<00:02,  5.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 67/80 [00:12<00:01,  7.14it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▋ | 69/80 [00:12<00:01,  6.34it/s]\u001b[A\n",
      "Evaluating workflow:  88%|████████▊ | 70/80 [00:12<00:01,  6.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 72/80 [00:12<00:01,  7.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▎| 74/80 [00:13<00:01,  4.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 76/80 [00:14<00:00,  4.28it/s]\u001b[A\n",
      "Evaluating workflow:  96%|█████████▋| 77/80 [00:14<00:00,  4.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 78/80 [00:14<00:00,  5.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 80/80 [00:14<00:00,  5.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2025-12-27 17:23:43.323\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 3 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.2875}\u001b[0m\n",
      "\u001b[32m2025-12-27 17:23:43.324\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.325}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 15%|█▌        | 3/20 [05:46<31:37, 111.60s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:23:43.327\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:23:54.657\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:25:07.833\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:25:12.767\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:25:12.767\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 4 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:01<01:43,  1.31s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   2%|▎         | 2/80 [00:02<01:35,  1.22s/it]\u001b[A\n",
      "Evaluating workflow:   5%|▌         | 4/80 [00:02<00:39,  1.94it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▋         | 5/80 [00:02<00:29,  2.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  11%|█▏        | 9/80 [00:03<00:12,  5.58it/s]\u001b[A\n",
      "Evaluating workflow:  14%|█▍        | 11/80 [00:03<00:10,  6.63it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  16%|█▋        | 13/80 [00:03<00:10,  6.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  20%|██        | 16/80 [00:03<00:09,  7.10it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██▏       | 17/80 [00:04<00:11,  5.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▌       | 20/80 [00:04<00:08,  7.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  26%|██▋       | 21/80 [00:05<00:12,  4.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 24/80 [00:05<00:11,  5.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  31%|███▏      | 25/80 [00:06<00:12,  4.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  34%|███▍      | 27/80 [00:06<00:11,  4.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  36%|███▋      | 29/80 [00:06<00:09,  5.38it/s]\u001b[A\n",
      "Evaluating workflow:  40%|████      | 32/80 [00:06<00:06,  7.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▎     | 34/80 [00:07<00:06,  7.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▌     | 36/80 [00:07<00:05,  8.01it/s]\u001b[A\n",
      "Evaluating workflow:  48%|████▊     | 38/80 [00:07<00:04,  8.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  50%|█████     | 40/80 [00:07<00:05,  7.62it/s]\u001b[A\n",
      "Evaluating workflow:  51%|█████▏    | 41/80 [00:07<00:05,  7.17it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▎    | 42/80 [00:08<00:08,  4.63it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 43/80 [00:08<00:08,  4.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▋    | 45/80 [00:09<00:06,  5.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  57%|█████▊    | 46/80 [00:09<00:06,  4.97it/s]\u001b[A\n",
      "Evaluating workflow:  60%|██████    | 48/80 [00:09<00:04,  6.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████▏   | 49/80 [00:09<00:05,  5.38it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▎   | 50/80 [00:10<00:06,  4.43it/s]\u001b[A\n",
      "Evaluating workflow:  66%|██████▋   | 53/80 [00:10<00:03,  7.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▉   | 55/80 [00:10<00:02,  9.38it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  71%|███████▏  | 57/80 [00:10<00:02,  9.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▌  | 60/80 [00:11<00:02,  7.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 62/80 [00:11<00:02,  6.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 64/80 [00:11<00:02,  6.39it/s]\u001b[A\n",
      "Evaluating workflow:  82%|████████▎ | 66/80 [00:11<00:01,  7.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  85%|████████▌ | 68/80 [00:12<00:02,  5.57it/s]\u001b[A\n",
      "Evaluating workflow:  89%|████████▉ | 71/80 [00:12<00:01,  7.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 73/80 [00:13<00:01,  5.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 76/80 [00:13<00:00,  6.68it/s]\u001b[A\n",
      "Evaluating workflow:  96%|█████████▋| 77/80 [00:13<00:00,  6.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 78/80 [00:13<00:00,  6.93it/s]\u001b[A\n",
      "Evaluating workflow:  99%|█████████▉| 79/80 [00:14<00:00,  6.59it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 80/80 [00:16<00:00,  4.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-27 17:25:28.942\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 4 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.275}\u001b[0m\n",
      "\u001b[32m2025-12-27 17:25:28.943\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.325}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 20%|██        | 4/20 [07:32<29:07, 109.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:25:28.946\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:25:38.737\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:26:48.790\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:26:53.961\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:26:53.961\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 5 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:00<01:18,  1.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   2%|▎         | 2/80 [00:02<01:26,  1.10s/it]\u001b[A\n",
      "Evaluating workflow:   5%|▌         | 4/80 [00:02<00:35,  2.11it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▋         | 5/80 [00:02<00:27,  2.70it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  11%|█▏        | 9/80 [00:02<00:11,  5.97it/s]\u001b[A\n",
      "Evaluating workflow:  14%|█▍        | 11/80 [00:02<00:09,  7.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  16%|█▋        | 13/80 [00:03<00:09,  7.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  20%|██        | 16/80 [00:03<00:07,  8.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  22%|██▎       | 18/80 [00:03<00:06,  9.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▌       | 20/80 [00:03<00:07,  8.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 22/80 [00:04<00:07,  7.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▉       | 23/80 [00:04<00:12,  4.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  31%|███▏      | 25/80 [00:05<00:10,  5.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  32%|███▎      | 26/80 [00:05<00:11,  4.51it/s]\u001b[A\n",
      "Evaluating workflow:  35%|███▌      | 28/80 [00:05<00:08,  5.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  39%|███▉      | 31/80 [00:05<00:06,  7.10it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 32/80 [00:06<00:07,  6.50it/s]\u001b[A\n",
      "Evaluating workflow:  41%|████▏     | 33/80 [00:06<00:09,  5.12it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 38/80 [00:06<00:05,  7.13it/s]\u001b[A\n",
      "Evaluating workflow:  50%|█████     | 40/80 [00:07<00:06,  6.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▎    | 42/80 [00:07<00:05,  7.05it/s]\u001b[A\n",
      "Evaluating workflow:  54%|█████▍    | 43/80 [00:07<00:05,  6.74it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▋    | 45/80 [00:07<00:04,  8.07it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  57%|█████▊    | 46/80 [00:08<00:04,  7.80it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▉    | 47/80 [00:08<00:06,  4.91it/s]\u001b[A\n",
      "Evaluating workflow:  60%|██████    | 48/80 [00:08<00:05,  5.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▌   | 52/80 [00:08<00:02, 10.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▉   | 55/80 [00:09<00:04,  5.94it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  71%|███████▏  | 57/80 [00:10<00:04,  4.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▌  | 60/80 [00:10<00:03,  6.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 62/80 [00:10<00:02,  6.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 64/80 [00:10<00:02,  7.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 67/80 [00:11<00:02,  6.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  85%|████████▌ | 68/80 [00:11<00:02,  5.98it/s]\u001b[A\n",
      "Evaluating workflow:  88%|████████▊ | 70/80 [00:11<00:01,  7.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 72/80 [00:12<00:01,  7.64it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 73/80 [00:12<00:01,  6.98it/s]\u001b[A\n",
      "Evaluating workflow:  92%|█████████▎| 74/80 [00:12<00:00,  6.44it/s]\u001b[A\n",
      "Evaluating workflow:  95%|█████████▌| 76/80 [00:12<00:00,  8.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 79/80 [00:14<00:00,  3.41it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 80/80 [00:14<00:00,  5.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2025-12-27 17:27:08.612\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 5 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.325}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 25%|██▌       | 5/20 [09:12<26:26, 105.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:27:08.613\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:27:19.154\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:28:31.198\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:28:36.030\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:28:36.030\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 6 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:01<01:20,  1.02s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   2%|▎         | 2/80 [00:01<01:05,  1.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 3/80 [00:02<00:57,  1.35it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   5%|▌         | 4/80 [00:02<00:47,  1.60it/s]\u001b[A\n",
      "Evaluating workflow:   9%|▉         | 7/80 [00:02<00:19,  3.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  14%|█▍        | 11/80 [00:03<00:10,  6.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  16%|█▋        | 13/80 [00:03<00:09,  7.39it/s]\u001b[A\n",
      "Evaluating workflow:  19%|█▉        | 15/80 [00:03<00:07,  8.43it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██▏       | 17/80 [00:03<00:06,  9.50it/s]\u001b[A\n",
      "Evaluating workflow:  25%|██▌       | 20/80 [00:03<00:04, 12.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 22/80 [00:04<00:06,  8.68it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 24/80 [00:04<00:05,  9.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  32%|███▎      | 26/80 [00:05<00:13,  3.92it/s]\u001b[A\n",
      "Evaluating workflow:  35%|███▌      | 28/80 [00:05<00:10,  4.85it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 30/80 [00:05<00:08,  6.11it/s]\u001b[A\n",
      "Evaluating workflow:  40%|████      | 32/80 [00:06<00:06,  7.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▎     | 34/80 [00:06<00:07,  6.00it/s]\u001b[A\n",
      "Evaluating workflow:  45%|████▌     | 36/80 [00:06<00:05,  7.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 38/80 [00:06<00:05,  7.86it/s]\u001b[A\n",
      "Evaluating workflow:  50%|█████     | 40/80 [00:06<00:04,  9.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▎    | 42/80 [00:07<00:04,  8.61it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▋    | 45/80 [00:07<00:03, 10.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▉    | 47/80 [00:07<00:03,  9.48it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████▏   | 49/80 [00:08<00:05,  5.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▎   | 50/80 [00:08<00:06,  4.62it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▌   | 52/80 [00:09<00:06,  4.65it/s]\u001b[A\n",
      "Evaluating workflow:  66%|██████▋   | 53/80 [00:09<00:05,  5.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▉   | 55/80 [00:09<00:03,  6.32it/s]\u001b[A\n",
      "Evaluating workflow:  71%|███████▏  | 57/80 [00:09<00:03,  7.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▎  | 58/80 [00:09<00:03,  7.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  74%|███████▍  | 59/80 [00:10<00:03,  5.57it/s]\u001b[A\n",
      "Evaluating workflow:  75%|███████▌  | 60/80 [00:10<00:03,  6.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▉  | 63/80 [00:10<00:01,  9.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████▏ | 65/80 [00:10<00:01,  9.36it/s]\u001b[A\n",
      "Evaluating workflow:  85%|████████▌ | 68/80 [00:10<00:01, 11.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 70/80 [00:11<00:01,  8.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 72/80 [00:11<00:01,  5.48it/s]\u001b[A\n",
      "Evaluating workflow:  91%|█████████▏| 73/80 [00:12<00:01,  5.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▎| 74/80 [00:12<00:01,  5.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 75/80 [00:12<00:01,  4.10it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 76/80 [00:13<00:01,  3.39it/s]\u001b[A\n",
      "Evaluating workflow:  98%|█████████▊| 78/80 [00:13<00:00,  4.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 80/80 [00:14<00:00,  5.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2025-12-27 17:28:50.681\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 6 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.3625}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 30%|███       | 6/20 [10:54<24:23, 104.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:28:50.682\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:29:01.763\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:30:09.515\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:30:14.706\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:30:14.707\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 7 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:00<00:50,  1.55it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▎         | 2/80 [00:00<00:29,  2.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[A\n",
      "Evaluating workflow:   8%|▊         | 6/80 [00:00<00:08,  8.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  10%|█         | 8/80 [00:01<00:09,  7.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▎        | 10/80 [00:01<00:09,  7.17it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  15%|█▌        | 12/80 [00:01<00:10,  6.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 14/80 [00:02<00:09,  6.71it/s]\u001b[A\n",
      "Evaluating workflow:  20%|██        | 16/80 [00:02<00:07,  8.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  22%|██▎       | 18/80 [00:02<00:06,  9.94it/s]\u001b[A\n",
      "Evaluating workflow:  25%|██▌       | 20/80 [00:02<00:05, 11.58it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▉       | 23/80 [00:02<00:04, 12.05it/s]\u001b[A\n",
      "Evaluating workflow:  31%|███▏      | 25/80 [00:02<00:04, 13.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  34%|███▍      | 27/80 [00:03<00:03, 13.69it/s]\u001b[A\n",
      "Evaluating workflow:  36%|███▋      | 29/80 [00:03<00:03, 14.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  39%|███▉      | 31/80 [00:03<00:03, 12.57it/s]\u001b[A\n",
      "Evaluating workflow:  41%|████▏     | 33/80 [00:03<00:03, 12.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  46%|████▋     | 37/80 [00:03<00:02, 15.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  50%|█████     | 40/80 [00:03<00:02, 17.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▎    | 42/80 [00:04<00:03, 11.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  57%|█████▊    | 46/80 [00:04<00:02, 13.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 48/80 [00:04<00:02, 11.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▎   | 50/80 [00:05<00:03,  8.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 54/80 [00:05<00:02, 11.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  70%|███████   | 56/80 [00:05<00:02, 11.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▎  | 58/80 [00:05<00:02,  9.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▌  | 60/80 [00:06<00:02,  9.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▎ | 66/80 [00:06<00:01, 13.73it/s]\u001b[A\n",
      "Evaluating workflow:  86%|████████▋ | 69/80 [00:06<00:00, 15.17it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▉ | 71/80 [00:06<00:00, 15.43it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 73/80 [00:07<00:00,  7.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 75/80 [00:07<00:00,  7.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▋| 77/80 [00:08<00:00,  4.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 78/80 [00:08<00:00,  3.90it/s]\u001b[A\n",
      "Evaluating workflow:  99%|█████████▉| 79/80 [00:09<00:00,  4.11it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 80/80 [00:09<00:00,  8.38it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-27 17:30:24.377\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 7 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.6375}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 35%|███▌      | 7/20 [12:27<21:52, 100.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:30:24.378\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:30:36.387\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:31:50.158\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:31:54.808\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:31:54.808\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 8 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:00<01:04,  1.23it/s]\u001b[A\n",
      "Evaluating workflow:   6%|▋         | 5/80 [00:00<00:11,  6.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   9%|▉         | 7/80 [00:01<00:12,  5.68it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  11%|█▏        | 9/80 [00:01<00:12,  5.91it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 11/80 [00:01<00:09,  7.51it/s]\u001b[A\n",
      "Evaluating workflow:  16%|█▋        | 13/80 [00:02<00:10,  6.26it/s]\u001b[A\n",
      "Evaluating workflow:  19%|█▉        | 15/80 [00:02<00:08,  7.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██▏       | 17/80 [00:02<00:07,  8.38it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  26%|██▋       | 21/80 [00:02<00:05, 10.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▉       | 23/80 [00:03<00:06,  9.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  31%|███▏      | 25/80 [00:03<00:06,  9.12it/s]\u001b[A\n",
      "Evaluating workflow:  38%|███▊      | 30/80 [00:03<00:03, 13.40it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  40%|████      | 32/80 [00:03<00:03, 12.54it/s]\u001b[A\n",
      "Evaluating workflow:  42%|████▎     | 34/80 [00:03<00:03, 13.17it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▉     | 39/80 [00:04<00:02, 17.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████▏    | 41/80 [00:04<00:02, 13.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 43/80 [00:04<00:03,  9.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▋    | 45/80 [00:05<00:04,  7.30it/s]\u001b[A\n",
      "Evaluating workflow:  57%|█████▊    | 46/80 [00:05<00:04,  7.58it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▉    | 47/80 [00:05<00:05,  6.57it/s]\u001b[A\n",
      "Evaluating workflow:  60%|██████    | 48/80 [00:05<00:04,  6.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  64%|██████▍   | 51/80 [00:05<00:02, 10.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▋   | 53/80 [00:06<00:03,  7.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  70%|███████   | 56/80 [00:06<00:02,  9.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  74%|███████▍  | 59/80 [00:06<00:02, 10.26it/s]\u001b[A\n",
      "Evaluating workflow:  79%|███████▉  | 63/80 [00:06<00:01, 13.63it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████▏ | 65/80 [00:07<00:01, 10.58it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 67/80 [00:07<00:01,  9.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 70/80 [00:07<00:00, 10.54it/s]\u001b[A\n",
      "Evaluating workflow:  90%|█████████ | 72/80 [00:07<00:00, 11.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 75/80 [00:08<00:00, 12.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▋| 77/80 [00:09<00:00,  3.34it/s]\u001b[A\n",
      "Evaluating workflow:  98%|█████████▊| 78/80 [00:10<00:00,  3.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 80/80 [00:10<00:00,  7.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-27 17:32:05.474\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 8 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.6}\u001b[0m\n",
      "\u001b[32m2025-12-27 17:32:05.475\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.6375}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 40%|████      | 8/20 [14:08<20:12, 101.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:32:05.478\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:32:19.467\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:33:28.936\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:33:33.186\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:33:33.187\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 9 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:00<01:04,  1.22it/s]\u001b[A\n",
      "Evaluating workflow:   6%|▋         | 5/80 [00:00<00:11,  6.48it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   9%|▉         | 7/80 [00:01<00:11,  6.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  11%|█▏        | 9/80 [00:01<00:13,  5.34it/s]\u001b[A\n",
      "Evaluating workflow:  15%|█▌        | 12/80 [00:01<00:08,  8.17it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 14/80 [00:02<00:09,  6.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  20%|██        | 16/80 [00:02<00:09,  6.85it/s]\u001b[A\n",
      "Evaluating workflow:  25%|██▌       | 20/80 [00:02<00:05, 10.58it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▉       | 23/80 [00:02<00:04, 13.41it/s]\u001b[A\n",
      "Evaluating workflow:  32%|███▎      | 26/80 [00:02<00:03, 15.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  36%|███▋      | 29/80 [00:03<00:04, 10.79it/s]\u001b[A\n",
      "Evaluating workflow:  41%|████▏     | 33/80 [00:03<00:03, 14.27it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▌     | 36/80 [00:03<00:03, 14.38it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▉     | 39/80 [00:04<00:04,  9.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████▏    | 41/80 [00:04<00:04,  8.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 43/80 [00:04<00:04,  8.92it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▋    | 45/80 [00:05<00:04,  8.43it/s]\u001b[A\n",
      "Evaluating workflow:  61%|██████▏   | 49/80 [00:05<00:02, 12.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  64%|██████▍   | 51/80 [00:05<00:02, 10.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▋   | 53/80 [00:05<00:02,  9.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▉   | 55/80 [00:06<00:02,  9.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  71%|███████▏  | 57/80 [00:06<00:02,  9.08it/s]\u001b[A\n",
      "Evaluating workflow:  74%|███████▍  | 59/80 [00:06<00:02, 10.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  76%|███████▋  | 61/80 [00:06<00:01, 11.86it/s]\u001b[A\n",
      "Evaluating workflow:  80%|████████  | 64/80 [00:06<00:01, 15.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 67/80 [00:06<00:00, 15.82it/s]\u001b[A\n",
      "Evaluating workflow:  86%|████████▋ | 69/80 [00:06<00:00, 16.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▉ | 71/80 [00:07<00:00, 13.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 73/80 [00:07<00:01,  5.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 75/80 [00:08<00:00,  6.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▋| 77/80 [00:09<00:00,  3.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 80/80 [00:09<00:00,  8.38it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-27 17:33:42.864\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 9 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.55}\u001b[0m\n",
      "\u001b[32m2025-12-27 17:33:42.864\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.6375}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 45%|████▌     | 9/20 [15:46<18:18, 99.89s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:33:42.867\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:33:53.866\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:35:14.790\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:35:19.731\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:35:19.731\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 10 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:00<00:58,  1.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   5%|▌         | 4/80 [00:00<00:14,  5.11it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  11%|█▏        | 9/80 [00:01<00:06, 10.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  14%|█▍        | 11/80 [00:01<00:09,  7.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  16%|█▋        | 13/80 [00:01<00:08,  7.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██▏       | 17/80 [00:02<00:05, 10.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  24%|██▍       | 19/80 [00:02<00:06, 10.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  26%|██▋       | 21/80 [00:02<00:05, 10.85it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 24/80 [00:02<00:05, 10.93it/s]\u001b[A\n",
      "Evaluating workflow:  32%|███▎      | 26/80 [00:02<00:04, 11.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▌      | 28/80 [00:02<00:04, 12.80it/s]\u001b[A\n",
      "Evaluating workflow:  38%|███▊      | 30/80 [00:03<00:03, 13.40it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  40%|████      | 32/80 [00:03<00:03, 14.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▌     | 36/80 [00:03<00:02, 16.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 38/80 [00:03<00:03, 12.60it/s]\u001b[A\n",
      "Evaluating workflow:  51%|█████▏    | 41/80 [00:03<00:02, 14.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▋    | 45/80 [00:03<00:01, 18.97it/s]\u001b[A\n",
      "Evaluating workflow:  60%|██████    | 48/80 [00:04<00:01, 19.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  64%|██████▍   | 51/80 [00:04<00:02, 10.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 54/80 [00:04<00:02, 11.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  71%|███████▏  | 57/80 [00:05<00:02,  9.60it/s]\u001b[A\n",
      "Evaluating workflow:  74%|███████▍  | 59/80 [00:05<00:01, 10.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 62/80 [00:05<00:01, 10.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 67/80 [00:05<00:00, 13.15it/s]\u001b[A\n",
      "Evaluating workflow:  88%|████████▊ | 70/80 [00:06<00:00, 13.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 73/80 [00:06<00:00, 13.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 75/80 [00:06<00:00, 12.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▋| 77/80 [00:06<00:00, 12.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 79/80 [00:06<00:00, 11.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|██████████| 80/80 [00:07<00:00, 10.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-27 17:35:27.167\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 10 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.6625}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 50%|█████     | 10/20 [17:30<16:52, 101.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:35:27.168\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:35:39.375\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:36:53.657\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:36:57.978\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:36:57.979\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 11 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:00<01:01,  1.28it/s]\u001b[A\n",
      "Evaluating workflow:   5%|▌         | 4/80 [00:00<00:13,  5.62it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   8%|▊         | 6/80 [00:01<00:19,  3.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  11%|█▏        | 9/80 [00:02<00:15,  4.52it/s]\u001b[A\n",
      "Evaluating workflow:  15%|█▌        | 12/80 [00:02<00:09,  6.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 14/80 [00:02<00:09,  6.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██▏       | 17/80 [00:02<00:06,  9.18it/s]\u001b[A\n",
      "Evaluating workflow:  25%|██▌       | 20/80 [00:02<00:05, 11.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 22/80 [00:03<00:05, 10.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 24/80 [00:03<00:05, 10.49it/s]\u001b[A\n",
      "Evaluating workflow:  34%|███▍      | 27/80 [00:03<00:04, 12.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  36%|███▋      | 29/80 [00:03<00:04, 12.07it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  39%|███▉      | 31/80 [00:03<00:04, 10.07it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████▏     | 33/80 [00:04<00:05,  9.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 35/80 [00:04<00:06,  7.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▌     | 36/80 [00:04<00:06,  6.45it/s]\u001b[A\n",
      "Evaluating workflow:  46%|████▋     | 37/80 [00:04<00:06,  6.82it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 38/80 [00:05<00:06,  6.38it/s]\u001b[A\n",
      "Evaluating workflow:  50%|█████     | 40/80 [00:05<00:04,  8.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████▏    | 41/80 [00:05<00:04,  8.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▋    | 45/80 [00:05<00:03, 10.74it/s]\u001b[A\n",
      "Evaluating workflow:  60%|██████    | 48/80 [00:05<00:02, 12.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  64%|██████▍   | 51/80 [00:05<00:01, 15.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▋   | 53/80 [00:06<00:02, 10.08it/s]\u001b[A\n",
      "Evaluating workflow:  70%|███████   | 56/80 [00:06<00:01, 12.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▎  | 58/80 [00:06<00:01, 11.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▌  | 60/80 [00:07<00:02,  9.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 62/80 [00:07<00:02,  8.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 64/80 [00:07<00:01,  8.90it/s]\u001b[A\n",
      "Evaluating workflow:  84%|████████▍ | 67/80 [00:07<00:01, 11.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▋ | 69/80 [00:07<00:01,  9.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▉ | 71/80 [00:08<00:01,  7.42it/s]\u001b[A\n",
      "Evaluating workflow:  91%|█████████▏| 73/80 [00:08<00:00,  8.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 75/80 [00:08<00:00,  9.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▋| 77/80 [00:08<00:00,  9.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 79/80 [00:09<00:00,  6.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 80/80 [00:09<00:00,  8.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2025-12-27 17:37:08.043\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 11 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.5875}\u001b[0m\n",
      "\u001b[32m2025-12-27 17:37:08.043\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.6625}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 55%|█████▌    | 11/20 [19:11<15:10, 101.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:37:08.047\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:37:17.191\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:38:31.333\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:38:36.231\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:38:36.232\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 12 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:00<00:51,  1.52it/s]\u001b[A\n",
      "Evaluating workflow:   4%|▍         | 3/80 [00:00<00:18,  4.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   8%|▊         | 6/80 [00:01<00:16,  4.61it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   9%|▉         | 7/80 [00:01<00:17,  4.21it/s]\u001b[A\n",
      "Evaluating workflow:  12%|█▎        | 10/80 [00:01<00:09,  7.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  15%|█▌        | 12/80 [00:02<00:09,  6.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 14/80 [00:02<00:11,  5.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  24%|██▍       | 19/80 [00:02<00:06,  8.74it/s]\u001b[A\n",
      "Evaluating workflow:  26%|██▋       | 21/80 [00:03<00:06,  9.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▉       | 23/80 [00:03<00:05, 10.04it/s]\u001b[A\n",
      "Evaluating workflow:  31%|███▏      | 25/80 [00:03<00:04, 11.17it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  36%|███▋      | 29/80 [00:03<00:03, 14.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████▏     | 33/80 [00:04<00:04, 11.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  46%|████▋     | 37/80 [00:04<00:03, 12.15it/s]\u001b[A\n",
      "Evaluating workflow:  49%|████▉     | 39/80 [00:04<00:03, 12.17it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████▏    | 41/80 [00:04<00:03, 12.85it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 43/80 [00:04<00:03, 10.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▋    | 45/80 [00:05<00:03,  9.11it/s]\u001b[A\n",
      "Evaluating workflow:  59%|█████▉    | 47/80 [00:05<00:03,  9.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████▏   | 49/80 [00:05<00:02, 10.51it/s]\u001b[A\n",
      "Evaluating workflow:  64%|██████▍   | 51/80 [00:05<00:02, 11.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▋   | 53/80 [00:05<00:02, 12.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▉   | 55/80 [00:06<00:02, 10.80it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  71%|███████▏  | 57/80 [00:06<00:02,  8.75it/s]\u001b[A\n",
      "Evaluating workflow:  75%|███████▌  | 60/80 [00:06<00:01, 11.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 62/80 [00:06<00:01, 12.10it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 64/80 [00:06<00:01, 10.71it/s]\u001b[A\n",
      "Evaluating workflow:  84%|████████▍ | 67/80 [00:07<00:01, 12.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 70/80 [00:07<00:00, 12.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 72/80 [00:07<00:00, 12.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▎| 74/80 [00:07<00:00, 10.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 76/80 [00:08<00:00,  5.35it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 78/80 [00:09<00:00,  5.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 80/80 [00:09<00:00,  8.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-27 17:38:46.366\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 12 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.6625}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 60%|██████    | 12/20 [20:49<13:22, 100.28s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:38:46.367\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:38:55.509\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:40:17.376\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:40:22.657\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:40:22.657\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 13 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:00<00:53,  1.47it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▎         | 2/80 [00:00<00:29,  2.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   9%|▉         | 7/80 [00:01<00:09,  7.58it/s]\u001b[A\n",
      "Evaluating workflow:  11%|█▏        | 9/80 [00:01<00:08,  8.34it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▎        | 10/80 [00:01<00:10,  6.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  14%|█▍        | 11/80 [00:02<00:14,  4.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  16%|█▋        | 13/80 [00:02<00:14,  4.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 14/80 [00:02<00:14,  4.50it/s]\u001b[A\n",
      "Evaluating workflow:  25%|██▌       | 20/80 [00:02<00:05, 10.27it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▉       | 23/80 [00:03<00:04, 12.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  31%|███▏      | 25/80 [00:03<00:06,  8.22it/s]\u001b[A\n",
      "Evaluating workflow:  36%|███▋      | 29/80 [00:03<00:04, 11.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▎     | 34/80 [00:03<00:02, 16.70it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  46%|████▋     | 37/80 [00:04<00:03, 11.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  50%|█████     | 40/80 [00:04<00:03, 11.84it/s]\u001b[A\n",
      "Evaluating workflow:  52%|█████▎    | 42/80 [00:04<00:02, 12.82it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▋    | 45/80 [00:05<00:03, 10.12it/s]\u001b[A\n",
      "Evaluating workflow:  59%|█████▉    | 47/80 [00:05<00:03, 10.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████▏   | 49/80 [00:05<00:03,  8.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  64%|██████▍   | 51/80 [00:06<00:03,  7.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▉   | 55/80 [00:06<00:02,  8.89it/s]\u001b[A\n",
      "Evaluating workflow:  72%|███████▎  | 58/80 [00:06<00:02, 10.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▌  | 60/80 [00:06<00:01, 10.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 62/80 [00:06<00:01, 10.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 64/80 [00:07<00:01, 11.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▎ | 66/80 [00:07<00:01, 11.08it/s]\u001b[A\n",
      "Evaluating workflow:  85%|████████▌ | 68/80 [00:07<00:01, 11.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 70/80 [00:07<00:00, 10.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 72/80 [00:08<00:01,  6.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 73/80 [00:08<00:01,  3.94it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 75/80 [00:09<00:01,  4.59it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 76/80 [00:09<00:00,  4.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▋| 77/80 [00:10<00:00,  3.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 78/80 [00:10<00:00,  3.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 79/80 [00:10<00:00,  2.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 80/80 [00:11<00:00,  6.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-27 17:40:34.314\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 13 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.6125}\u001b[0m\n",
      "\u001b[32m2025-12-27 17:40:34.314\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.6625}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 65%|██████▌   | 13/20 [22:37<11:58, 102.60s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:40:34.318\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:40:45.860\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:41:58.544\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:42:05.065\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:42:05.066\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 14 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:00<00:54,  1.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 3/80 [00:00<00:21,  3.59it/s]\u001b[A\n",
      "Evaluating workflow:  10%|█         | 8/80 [00:01<00:07, 10.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▎        | 10/80 [00:01<00:10,  6.98it/s]\u001b[A\n",
      "Evaluating workflow:  15%|█▌        | 12/80 [00:01<00:07,  8.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 14/80 [00:01<00:07,  9.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  20%|██        | 16/80 [00:02<00:07,  8.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▌       | 20/80 [00:02<00:06,  8.73it/s]\u001b[A\n",
      "Evaluating workflow:  28%|██▊       | 22/80 [00:02<00:06,  9.63it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 24/80 [00:02<00:05, 10.89it/s]\u001b[A\n",
      "Evaluating workflow:  32%|███▎      | 26/80 [00:03<00:04, 11.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▌      | 28/80 [00:03<00:04, 12.57it/s]\u001b[A\n",
      "Evaluating workflow:  38%|███▊      | 30/80 [00:03<00:03, 13.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  40%|████      | 32/80 [00:03<00:03, 12.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▎     | 34/80 [00:03<00:05,  8.64it/s]\u001b[A\n",
      "Evaluating workflow:  48%|████▊     | 38/80 [00:03<00:03, 13.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▎    | 42/80 [00:04<00:02, 15.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▋    | 45/80 [00:04<00:02, 14.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 48/80 [00:04<00:02, 12.08it/s]\u001b[A\n",
      "Evaluating workflow:  65%|██████▌   | 52/80 [00:04<00:01, 15.84it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▉   | 55/80 [00:05<00:01, 14.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  71%|███████▏  | 57/80 [00:05<00:02, 11.40it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  74%|███████▍  | 59/80 [00:05<00:02, 10.27it/s]\u001b[A\n",
      "Evaluating workflow:  78%|███████▊  | 62/80 [00:05<00:01, 12.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████▏ | 65/80 [00:06<00:01, 12.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 67/80 [00:06<00:01, 10.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▋ | 69/80 [00:06<00:01,  9.14it/s]\u001b[A\n",
      "Evaluating workflow:  90%|█████████ | 72/80 [00:06<00:00, 11.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▎| 74/80 [00:06<00:00, 12.41it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 76/80 [00:07<00:00,  5.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 78/80 [00:08<00:00,  4.92it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 80/80 [00:09<00:00,  8.62it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-27 17:42:14.448\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 14 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.65}\u001b[0m\n",
      "\u001b[32m2025-12-27 17:42:14.448\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.6625}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 70%|███████   | 14/20 [24:17<10:11, 101.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:42:14.451\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:42:26.287\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:43:37.558\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:43:41.495\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:43:41.495\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 15 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:00<01:04,  1.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   8%|▊         | 6/80 [00:01<00:12,  6.07it/s]\u001b[A\n",
      "Evaluating workflow:   9%|▉         | 7/80 [00:01<00:12,  5.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  10%|█         | 8/80 [00:01<00:15,  4.59it/s]\u001b[A\n",
      "Evaluating workflow:  11%|█▏        | 9/80 [00:01<00:14,  4.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  16%|█▋        | 13/80 [00:02<00:08,  7.81it/s]\u001b[A\n",
      "Evaluating workflow:  19%|█▉        | 15/80 [00:02<00:07,  8.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  20%|██        | 16/80 [00:02<00:08,  7.66it/s]\u001b[A\n",
      "Evaluating workflow:  21%|██▏       | 17/80 [00:02<00:08,  7.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▌       | 20/80 [00:02<00:05, 11.46it/s]\u001b[A\n",
      "Evaluating workflow:  28%|██▊       | 22/80 [00:02<00:04, 13.10it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 24/80 [00:03<00:04, 11.82it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  32%|███▎      | 26/80 [00:03<00:04, 10.92it/s]\u001b[A\n",
      "Evaluating workflow:  35%|███▌      | 28/80 [00:03<00:04, 12.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 30/80 [00:03<00:04, 12.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  40%|████      | 32/80 [00:03<00:05,  9.32it/s]\u001b[A\n",
      "Evaluating workflow:  42%|████▎     | 34/80 [00:04<00:04, 10.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  46%|████▋     | 37/80 [00:04<00:02, 14.45it/s]\u001b[A\n",
      "Evaluating workflow:  49%|████▉     | 39/80 [00:04<00:02, 14.40it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████▏    | 41/80 [00:04<00:02, 14.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 43/80 [00:04<00:04,  9.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▋    | 45/80 [00:04<00:03, 10.68it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▉    | 47/80 [00:05<00:03,  9.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████▏   | 49/80 [00:05<00:03,  9.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  64%|██████▍   | 51/80 [00:05<00:03,  8.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 54/80 [00:05<00:02, 10.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  70%|███████   | 56/80 [00:06<00:02,  9.87it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 59/80 [00:06<00:01, 11.38it/s]\u001b[A\n",
      "Evaluating workflow:  79%|███████▉  | 63/80 [00:06<00:01, 14.98it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████▏ | 65/80 [00:06<00:01, 11.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 67/80 [00:07<00:01, 11.62it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 70/80 [00:07<00:00, 10.58it/s]\u001b[A\n",
      "Evaluating workflow:  90%|█████████ | 72/80 [00:07<00:00, 11.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▎| 74/80 [00:07<00:00,  9.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 76/80 [00:08<00:00,  4.57it/s]\u001b[A\n",
      "Evaluating workflow:  96%|█████████▋| 77/80 [00:08<00:00,  4.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 78/80 [00:09<00:00,  5.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 80/80 [00:10<00:00,  7.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-27 17:43:51.681\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 15 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.6}\u001b[0m\n",
      "\u001b[32m2025-12-27 17:43:51.682\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.6625}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 75%|███████▌  | 15/20 [25:55<08:22, 100.46s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:43:51.685\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:44:01.260\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:45:19.016\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:45:23.596\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:45:23.597\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 16 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:00<01:04,  1.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▋         | 5/80 [00:01<00:15,  4.82it/s]\u001b[A\n",
      "Evaluating workflow:   8%|▊         | 6/80 [00:01<00:14,  5.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   9%|▉         | 7/80 [00:01<00:15,  4.63it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  14%|█▍        | 11/80 [00:02<00:10,  6.74it/s]\u001b[A\n",
      "Evaluating workflow:  16%|█▋        | 13/80 [00:02<00:08,  8.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▉        | 15/80 [00:02<00:09,  7.18it/s]\u001b[A\n",
      "Evaluating workflow:  22%|██▎       | 18/80 [00:02<00:06,  9.92it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▌       | 20/80 [00:02<00:05, 10.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 22/80 [00:03<00:05, 10.24it/s]\u001b[A\n",
      "Evaluating workflow:  30%|███       | 24/80 [00:03<00:04, 11.40it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▌      | 28/80 [00:03<00:03, 16.12it/s]\u001b[A\n",
      "Evaluating workflow:  38%|███▊      | 30/80 [00:03<00:03, 15.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▎     | 34/80 [00:03<00:02, 19.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  46%|████▋     | 37/80 [00:03<00:03, 12.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▉     | 39/80 [00:04<00:03, 10.83it/s]\u001b[A\n",
      "Evaluating workflow:  54%|█████▍    | 43/80 [00:04<00:02, 13.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▋    | 45/80 [00:04<00:03, 10.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▉    | 47/80 [00:05<00:04,  7.62it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████▏   | 49/80 [00:05<00:04,  7.55it/s]\u001b[A\n",
      "Evaluating workflow:  64%|██████▍   | 51/80 [00:05<00:03,  8.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 54/80 [00:05<00:02, 10.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▎  | 58/80 [00:06<00:01, 12.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  76%|███████▋  | 61/80 [00:06<00:01, 12.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 64/80 [00:06<00:01, 12.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▎ | 66/80 [00:06<00:01, 11.09it/s]\u001b[A\n",
      "Evaluating workflow:  85%|████████▌ | 68/80 [00:06<00:01, 11.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 70/80 [00:07<00:00, 12.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 73/80 [00:07<00:00,  9.35it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 75/80 [00:09<00:01,  3.72it/s]\u001b[A\n",
      "Evaluating workflow:  98%|█████████▊| 78/80 [00:09<00:00,  5.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 80/80 [00:09<00:00,  8.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-27 17:45:33.517\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 16 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.6375}\u001b[0m\n",
      "\u001b[32m2025-12-27 17:45:33.517\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.6625}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 80%|████████  | 16/20 [27:36<06:43, 100.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:45:33.520\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:45:45.429\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:46:56.755\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:47:01.016\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:47:01.016\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 17 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:00<00:55,  1.42it/s]\u001b[A\n",
      "Evaluating workflow:   4%|▍         | 3/80 [00:00<00:17,  4.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  10%|█         | 8/80 [00:01<00:08,  8.00it/s]\u001b[A\n",
      "Evaluating workflow:  12%|█▎        | 10/80 [00:01<00:07,  9.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  15%|█▌        | 12/80 [00:01<00:07,  9.31it/s]\u001b[A\n",
      "Evaluating workflow:  19%|█▉        | 15/80 [00:01<00:05, 11.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██▏       | 17/80 [00:02<00:07,  8.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  26%|██▋       | 21/80 [00:02<00:05, 10.18it/s]\u001b[A\n",
      "Evaluating workflow:  31%|███▏      | 25/80 [00:02<00:04, 13.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  34%|███▍      | 27/80 [00:02<00:04, 12.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  36%|███▋      | 29/80 [00:03<00:04, 11.38it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████▏     | 33/80 [00:03<00:03, 13.11it/s]\u001b[A\n",
      "Evaluating workflow:  48%|████▊     | 38/80 [00:03<00:02, 17.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  50%|█████     | 40/80 [00:03<00:02, 14.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▎    | 42/80 [00:03<00:03, 12.26it/s]\u001b[A\n",
      "Evaluating workflow:  56%|█████▋    | 45/80 [00:04<00:02, 14.34it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 48/80 [00:04<00:02, 15.59it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▎   | 50/80 [00:04<00:02, 13.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▌   | 52/80 [00:04<00:02, 10.22it/s]\u001b[A\n",
      "Evaluating workflow:  70%|███████   | 56/80 [00:04<00:01, 14.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▌  | 60/80 [00:05<00:01, 18.07it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▉  | 63/80 [00:05<00:01, 10.09it/s]\u001b[A\n",
      "Evaluating workflow:  81%|████████▏ | 65/80 [00:05<00:01, 10.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▋ | 69/80 [00:06<00:00, 12.48it/s]\u001b[A\n",
      "Evaluating workflow:  90%|█████████ | 72/80 [00:06<00:00, 13.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▎| 74/80 [00:06<00:00, 13.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 76/80 [00:06<00:00, 10.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 78/80 [00:07<00:00,  6.80it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 80/80 [00:08<00:00,  9.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-27 17:47:09.778\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 17 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.675}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 85%|████████▌ | 17/20 [29:13<04:58, 99.49s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:47:09.779\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:47:19.975\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:48:35.667\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:48:39.750\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:48:39.750\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 18 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:00<00:51,  1.53it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▎         | 2/80 [00:00<00:29,  2.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  11%|█▏        | 9/80 [00:00<00:05, 13.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  15%|█▌        | 12/80 [00:01<00:08,  8.01it/s]\u001b[A\n",
      "Evaluating workflow:  20%|██        | 16/80 [00:01<00:05, 11.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  24%|██▍       | 19/80 [00:01<00:04, 13.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 22/80 [00:02<00:04, 13.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 24/80 [00:02<00:05, 10.03it/s]\u001b[A\n",
      "Evaluating workflow:  35%|███▌      | 28/80 [00:02<00:03, 13.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  39%|███▉      | 31/80 [00:02<00:03, 14.74it/s]\u001b[A\n",
      "Evaluating workflow:  42%|████▎     | 34/80 [00:02<00:02, 15.41it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  46%|████▋     | 37/80 [00:03<00:03, 13.67it/s]\u001b[A\n",
      "Evaluating workflow:  50%|█████     | 40/80 [00:03<00:02, 15.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▎    | 42/80 [00:03<00:02, 15.01it/s]\u001b[A\n",
      "Evaluating workflow:  57%|█████▊    | 46/80 [00:03<00:01, 19.74it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▎   | 50/80 [00:03<00:01, 21.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▋   | 53/80 [00:04<00:01, 14.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▉   | 55/80 [00:04<00:02, 10.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  74%|███████▍  | 59/80 [00:04<00:01, 12.70it/s]\u001b[A\n",
      "Evaluating workflow:  76%|███████▋  | 61/80 [00:04<00:01, 13.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▉  | 63/80 [00:05<00:01, 13.52it/s]\u001b[A\n",
      "Evaluating workflow:  81%|████████▏ | 65/80 [00:05<00:01, 13.74it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 67/80 [00:05<00:01, 11.10it/s]\u001b[A\n",
      "Evaluating workflow:  88%|████████▊ | 70/80 [00:05<00:00, 13.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 72/80 [00:05<00:00, 11.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 75/80 [00:05<00:00, 14.24it/s]\u001b[A\n",
      "Evaluating workflow:  98%|█████████▊| 78/80 [00:06<00:00, 15.62it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 80/80 [00:06<00:00, 12.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-27 17:48:46.266\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 18 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.725}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 90%|█████████ | 18/20 [30:49<03:17, 98.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:48:46.267\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:48:55.571\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:50:09.638\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:50:14.730\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:50:14.731\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 19 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:00<00:57,  1.37it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▎         | 2/80 [00:00<00:31,  2.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   5%|▌         | 4/80 [00:01<00:14,  5.22it/s]\u001b[A\n",
      "Evaluating workflow:   9%|▉         | 7/80 [00:01<00:08,  8.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  11%|█▏        | 9/80 [00:01<00:07,  9.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  14%|█▍        | 11/80 [00:01<00:11,  5.96it/s]\u001b[A\n",
      "Evaluating workflow:  19%|█▉        | 15/80 [00:02<00:06,  9.70it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██▏       | 17/80 [00:02<00:06,  9.58it/s]\u001b[A\n",
      "Evaluating workflow:  24%|██▍       | 19/80 [00:02<00:06,  9.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  26%|██▋       | 21/80 [00:02<00:05, 11.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▉       | 23/80 [00:02<00:05, 11.11it/s]\u001b[A\n",
      "Evaluating workflow:  32%|███▎      | 26/80 [00:02<00:03, 13.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▌      | 28/80 [00:03<00:03, 13.70it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 30/80 [00:03<00:04, 11.09it/s]\u001b[A\n",
      "Evaluating workflow:  41%|████▏     | 33/80 [00:03<00:03, 13.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 35/80 [00:03<00:03, 13.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  46%|████▋     | 37/80 [00:03<00:03, 13.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  50%|█████     | 40/80 [00:04<00:03, 10.74it/s]\u001b[A\n",
      "Evaluating workflow:  52%|█████▎    | 42/80 [00:04<00:03, 12.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▌    | 44/80 [00:04<00:03, 11.86it/s]\u001b[A\n",
      "Evaluating workflow:  57%|█████▊    | 46/80 [00:04<00:02, 12.80it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 48/80 [00:04<00:03,  9.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▎   | 50/80 [00:05<00:03,  8.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▌   | 52/80 [00:05<00:03,  9.10it/s]\u001b[A\n",
      "Evaluating workflow:  69%|██████▉   | 55/80 [00:05<00:02, 10.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  71%|███████▏  | 57/80 [00:05<00:01, 12.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  76%|███████▋  | 61/80 [00:05<00:01, 13.34it/s]\u001b[A\n",
      "Evaluating workflow:  79%|███████▉  | 63/80 [00:06<00:01, 12.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  85%|████████▌ | 68/80 [00:06<00:00, 16.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 70/80 [00:06<00:00, 13.82it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▎| 74/80 [00:06<00:00, 16.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 76/80 [00:06<00:00, 13.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 78/80 [00:08<00:00,  4.94it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 80/80 [00:09<00:00,  8.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-27 17:50:24.114\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 19 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.65}\u001b[0m\n",
      "\u001b[32m2025-12-27 17:50:24.114\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.725}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 95%|█████████▌| 19/20 [32:27<01:38, 98.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:50:24.118\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:50:34.185\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:51:47.535\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2025-12-27 17:51:52.140\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2025-12-27 17:51:52.140\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 20 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/80 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 1/80 [00:00<00:58,  1.36it/s]\u001b[A\n",
      "Evaluating workflow:  12%|█▎        | 10/80 [00:00<00:04, 14.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 14/80 [00:01<00:07,  9.29it/s]\u001b[A\n",
      "Evaluating workflow:  24%|██▍       | 19/80 [00:01<00:04, 13.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 22/80 [00:02<00:05, 10.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 24/80 [00:02<00:05,  9.85it/s]\u001b[A\n",
      "Evaluating workflow:  32%|███▎      | 26/80 [00:02<00:05, 10.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▌      | 28/80 [00:02<00:05,  9.87it/s]\u001b[A\n",
      "Evaluating workflow:  40%|████      | 32/80 [00:02<00:03, 13.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▌     | 36/80 [00:03<00:02, 15.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████▏    | 41/80 [00:03<00:02, 18.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▌    | 44/80 [00:03<00:02, 17.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  57%|█████▊    | 46/80 [00:03<00:02, 13.36it/s]\u001b[A\n",
      "Evaluating workflow:  62%|██████▎   | 50/80 [00:04<00:01, 15.48it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▌   | 52/80 [00:04<00:01, 16.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 54/80 [00:04<00:01, 14.78it/s]\u001b[A\n",
      "Evaluating workflow:  70%|███████   | 56/80 [00:04<00:01, 15.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▎  | 58/80 [00:04<00:01, 14.35it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▌  | 60/80 [00:04<00:01, 10.78it/s]\u001b[A\n",
      "Evaluating workflow:  79%|███████▉  | 63/80 [00:04<00:01, 13.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▎ | 66/80 [00:05<00:00, 14.68it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  85%|████████▌ | 68/80 [00:05<00:01, 11.32it/s]\u001b[A\n",
      "Evaluating workflow:  88%|████████▊ | 70/80 [00:05<00:00, 10.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 73/80 [00:05<00:00, 13.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 75/80 [00:06<00:00,  9.44it/s]\u001b[A\n",
      "Evaluating workflow:  96%|█████████▋| 77/80 [00:06<00:00, 10.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 79/80 [00:06<00:00,  7.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|██████████| 80/80 [00:07<00:00, 10.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-27 17:52:00.210\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 20 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.7}\u001b[0m\n",
      "\u001b[32m2025-12-27 17:52:00.211\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.725}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 20/20 [34:03<00:00, 102.18s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 17:52:00.215\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m300\u001b[0m - \u001b[1mReached the maximum number of steps 20. Optimization has finished.\u001b[0m\n",
      "\u001b[32m2025-12-27 17:52:00.216\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.workflow.workflow_graph\u001b[0m:\u001b[36msave_module\u001b[0m:\u001b[36m1201\u001b[0m - \u001b[1mSaving SequentialWorkFlowGraph to ./PertQA_textgrad_final.json\u001b[0m\n",
      "\u001b[32m2025-12-27 17:52:00.217\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.workflow.workflow_graph\u001b[0m:\u001b[36msave_module\u001b[0m:\u001b[36m1201\u001b[0m - \u001b[1mSaving SequentialWorkFlowGraph to ./PertQA_textgrad_best.json\u001b[0m\n",
      "\u001b[32m2025-12-27 17:52:00.218\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mrestore_best_graph\u001b[0m:\u001b[36m448\u001b[0m - \u001b[1mRestored the best graph from snapshot with metrics {'f1': 0.0, 'em': 0.0, 'acc': 0.725}\u001b[0m\n",
      "\u001b[32m2025-12-27 17:52:00.218\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m10\u001b[0m - \u001b[1mEvaluating workflow on test set...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 1/2500 [00:00<14:36,  2.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 3/2500 [00:00<06:56,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 6/2500 [00:00<04:48,  8.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 13/2500 [00:01<03:39, 11.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 17/2500 [00:01<03:41, 11.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 20/2500 [00:01<02:52, 14.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 23/2500 [00:02<02:51, 14.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 28/2500 [00:02<03:12, 12.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|▏         | 36/2500 [00:02<01:55, 21.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 39/2500 [00:02<01:48, 22.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 42/2500 [00:03<02:12, 18.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 45/2500 [00:03<02:41, 15.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 47/2500 [00:03<03:17, 12.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 51/2500 [00:04<03:21, 12.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 56/2500 [00:04<02:51, 14.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 59/2500 [00:04<02:46, 14.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 63/2500 [00:05<03:50, 10.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 66/2500 [00:05<03:12, 12.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 71/2500 [00:05<02:48, 14.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 77/2500 [00:05<01:46, 22.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 80/2500 [00:05<02:17, 17.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 83/2500 [00:06<02:14, 17.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 88/2500 [00:06<02:39, 15.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 90/2500 [00:06<02:33, 15.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 94/2500 [00:06<02:39, 15.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 96/2500 [00:07<03:10, 12.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 98/2500 [00:07<04:07,  9.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 104/2500 [00:07<03:11, 12.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 110/2500 [00:08<02:28, 16.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 115/2500 [00:08<02:11, 18.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 118/2500 [00:08<02:43, 14.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 123/2500 [00:08<02:29, 15.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 128/2500 [00:09<02:08, 18.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 131/2500 [00:09<03:02, 13.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 136/2500 [00:09<02:36, 15.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 140/2500 [00:10<02:38, 14.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 142/2500 [00:10<02:47, 14.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 146/2500 [00:10<02:50, 13.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 150/2500 [00:10<03:07, 12.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 152/2500 [00:11<02:52, 13.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 161/2500 [00:11<02:00, 19.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 163/2500 [00:11<02:03, 18.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 165/2500 [00:11<02:31, 15.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 172/2500 [00:12<02:48, 13.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 178/2500 [00:12<02:27, 15.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 180/2500 [00:12<02:30, 15.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 182/2500 [00:13<03:16, 11.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 187/2500 [00:13<02:21, 16.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 189/2500 [00:13<02:49, 13.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 194/2500 [00:13<02:34, 14.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 198/2500 [00:13<02:02, 18.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 201/2500 [00:14<02:15, 17.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 206/2500 [00:14<02:32, 15.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 211/2500 [00:15<02:51, 13.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 213/2500 [00:15<03:41, 10.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 215/2500 [00:15<03:44, 10.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 219/2500 [00:15<02:47, 13.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 223/2500 [00:15<02:42, 14.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 229/2500 [00:16<02:34, 14.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 234/2500 [00:16<01:54, 19.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 240/2500 [00:16<02:12, 17.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 243/2500 [00:17<02:33, 14.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 247/2500 [00:17<02:36, 14.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 252/2500 [00:17<02:31, 14.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 256/2500 [00:18<02:20, 15.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 258/2500 [00:18<02:27, 15.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 262/2500 [00:18<03:31, 10.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 267/2500 [00:19<02:39, 14.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 269/2500 [00:19<03:14, 11.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 271/2500 [00:19<03:19, 11.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 279/2500 [00:19<02:04, 17.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 283/2500 [00:19<01:44, 21.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 286/2500 [00:20<01:59, 18.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 291/2500 [00:20<02:51, 12.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 297/2500 [00:21<02:28, 14.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 302/2500 [00:21<02:58, 12.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 309/2500 [00:21<02:12, 16.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 312/2500 [00:22<03:04, 11.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 319/2500 [00:22<02:20, 15.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 323/2500 [00:22<02:22, 15.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 328/2500 [00:23<02:08, 16.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 330/2500 [00:23<02:35, 13.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 336/2500 [00:23<02:43, 13.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 340/2500 [00:24<02:27, 14.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 347/2500 [00:24<01:42, 21.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 350/2500 [00:24<02:29, 14.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 352/2500 [00:25<03:08, 11.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 356/2500 [00:25<02:48, 12.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 362/2500 [00:25<02:39, 13.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 366/2500 [00:26<02:34, 13.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 369/2500 [00:26<02:20, 15.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 371/2500 [00:26<03:01, 11.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 376/2500 [00:26<02:24, 14.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 378/2500 [00:27<02:37, 13.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 384/2500 [00:27<02:22, 14.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 386/2500 [00:27<02:23, 14.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 392/2500 [00:27<02:35, 13.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 396/2500 [00:28<02:16, 15.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 400/2500 [00:28<02:08, 16.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 402/2500 [00:28<02:11, 15.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 404/2500 [00:28<02:49, 12.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 408/2500 [00:29<02:53, 12.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 414/2500 [00:29<02:03, 16.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 420/2500 [00:29<02:04, 16.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 425/2500 [00:30<02:22, 14.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 427/2500 [00:30<02:14, 15.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 431/2500 [00:30<02:26, 14.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 435/2500 [00:30<02:20, 14.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 438/2500 [00:31<01:59, 17.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 442/2500 [00:31<02:34, 13.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 449/2500 [00:31<01:41, 20.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 452/2500 [00:32<02:53, 11.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 456/2500 [00:32<02:40, 12.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 459/2500 [00:32<02:22, 14.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 463/2500 [00:33<03:02, 11.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 467/2500 [00:33<02:53, 11.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 470/2500 [00:33<02:15, 15.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 477/2500 [00:34<02:23, 14.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 479/2500 [00:34<02:57, 11.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 483/2500 [00:34<02:22, 14.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 491/2500 [00:34<01:42, 19.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 498/2500 [00:35<02:12, 15.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 501/2500 [00:35<02:01, 16.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 504/2500 [00:35<02:05, 15.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 506/2500 [00:36<03:02, 10.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 510/2500 [00:36<02:25, 13.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 517/2500 [00:36<02:04, 15.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 519/2500 [00:37<02:26, 13.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 524/2500 [00:37<02:40, 12.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 530/2500 [00:37<02:12, 14.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 534/2500 [00:38<02:11, 14.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 539/2500 [00:38<01:43, 18.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 542/2500 [00:38<02:14, 14.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 548/2500 [00:39<01:58, 16.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 551/2500 [00:39<01:43, 18.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 554/2500 [00:39<02:33, 12.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 561/2500 [00:40<02:06, 15.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 564/2500 [00:40<01:53, 17.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 567/2500 [00:40<02:28, 12.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 573/2500 [00:40<02:11, 14.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 575/2500 [00:41<02:35, 12.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 577/2500 [00:41<02:34, 12.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 581/2500 [00:41<02:24, 13.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 583/2500 [00:41<02:14, 14.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 587/2500 [00:41<02:09, 14.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 593/2500 [00:42<02:18, 13.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 601/2500 [00:42<01:27, 21.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 604/2500 [00:42<01:44, 18.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 607/2500 [00:43<02:37, 12.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 611/2500 [00:43<02:33, 12.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 615/2500 [00:43<02:16, 13.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 619/2500 [00:44<01:50, 16.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 622/2500 [00:44<01:49, 17.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 624/2500 [00:44<02:08, 14.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 630/2500 [00:44<01:56, 16.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 632/2500 [00:45<02:35, 12.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 638/2500 [00:45<02:04, 14.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 640/2500 [00:45<02:18, 13.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 644/2500 [00:46<02:19, 13.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 646/2500 [00:46<02:07, 14.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 651/2500 [00:46<02:30, 12.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 653/2500 [00:46<02:22, 12.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 655/2500 [00:46<02:30, 12.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 660/2500 [00:47<02:15, 13.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 663/2500 [00:47<02:07, 14.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 665/2500 [00:47<02:24, 12.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 670/2500 [00:47<01:57, 15.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 672/2500 [00:48<02:25, 12.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 682/2500 [00:48<01:41, 17.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 685/2500 [00:49<02:35, 11.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 691/2500 [00:49<02:16, 13.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 693/2500 [00:49<02:55, 10.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 700/2500 [00:50<01:51, 16.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 702/2500 [00:50<02:14, 13.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 706/2500 [00:50<02:23, 12.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 714/2500 [00:51<01:35, 18.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 717/2500 [00:51<02:33, 11.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 724/2500 [00:51<01:40, 17.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 727/2500 [00:52<01:51, 15.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 730/2500 [00:52<02:33, 11.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 735/2500 [00:52<01:48, 16.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 738/2500 [00:52<01:55, 15.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 743/2500 [00:53<02:16, 12.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 745/2500 [00:53<02:34, 11.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 750/2500 [00:53<02:14, 13.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 752/2500 [00:53<02:04, 14.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 756/2500 [00:54<02:07, 13.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 760/2500 [00:54<01:39, 17.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 764/2500 [00:54<01:38, 17.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 766/2500 [00:55<02:29, 11.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 772/2500 [00:55<01:53, 15.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 777/2500 [00:55<01:23, 20.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 780/2500 [00:55<01:59, 14.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 782/2500 [00:56<02:39, 10.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 789/2500 [00:56<02:03, 13.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 791/2500 [00:56<02:13, 12.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 795/2500 [00:56<01:41, 16.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 798/2500 [00:57<02:18, 12.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 804/2500 [00:57<01:59, 14.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 806/2500 [00:57<02:13, 12.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 811/2500 [00:58<01:46, 15.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 815/2500 [00:58<01:26, 19.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 818/2500 [00:58<02:15, 12.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 825/2500 [00:59<01:39, 16.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 828/2500 [00:59<02:19, 12.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 832/2500 [00:59<02:13, 12.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 835/2500 [01:00<02:16, 12.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 839/2500 [01:00<02:16, 12.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 841/2500 [01:00<02:25, 11.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 846/2500 [01:00<01:52, 14.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 850/2500 [01:01<01:53, 14.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 853/2500 [01:01<01:43, 15.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 858/2500 [01:01<01:57, 14.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 860/2500 [01:01<02:21, 11.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 866/2500 [01:02<01:47, 15.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 869/2500 [01:02<01:32, 17.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 873/2500 [01:02<02:28, 10.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 876/2500 [01:03<01:57, 13.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 880/2500 [01:03<02:14, 12.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 885/2500 [01:03<02:02, 13.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 891/2500 [01:04<01:39, 16.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 895/2500 [01:04<01:45, 15.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 900/2500 [01:04<01:44, 15.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 904/2500 [01:05<02:33, 10.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 911/2500 [01:05<01:40, 15.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 915/2500 [01:06<02:17, 11.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 918/2500 [01:06<02:09, 12.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 920/2500 [01:06<02:08, 12.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 925/2500 [01:06<01:44, 15.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 927/2500 [01:06<01:58, 13.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 933/2500 [01:07<01:46, 14.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 935/2500 [01:07<01:47, 14.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 939/2500 [01:07<01:56, 13.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 945/2500 [01:08<01:25, 18.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 948/2500 [01:08<02:21, 10.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 953/2500 [01:09<02:17, 11.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 958/2500 [01:09<01:44, 14.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 960/2500 [01:09<02:20, 11.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 962/2500 [01:10<03:01,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 969/2500 [01:10<02:46,  9.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 971/2500 [01:11<02:30, 10.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 975/2500 [01:11<02:22, 10.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 980/2500 [01:11<01:49, 13.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 985/2500 [01:11<01:44, 14.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 989/2500 [01:12<01:43, 14.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 993/2500 [01:12<02:05, 12.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 997/2500 [01:12<01:52, 13.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 999/2500 [01:13<02:13, 11.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1006/2500 [01:13<01:34, 15.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1011/2500 [01:13<01:32, 16.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1013/2500 [01:14<01:53, 13.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1020/2500 [01:14<01:28, 16.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1024/2500 [01:14<01:40, 14.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1026/2500 [01:14<01:40, 14.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1030/2500 [01:15<01:39, 14.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████▏     | 1036/2500 [01:15<01:34, 15.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1041/2500 [01:15<01:30, 16.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1043/2500 [01:15<01:33, 15.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1045/2500 [01:16<01:40, 14.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1047/2500 [01:16<01:54, 12.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1049/2500 [01:16<02:30,  9.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1055/2500 [01:17<02:01, 11.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1059/2500 [01:17<01:40, 14.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1061/2500 [01:17<01:54, 12.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1065/2500 [01:17<01:35, 15.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1070/2500 [01:18<01:37, 14.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1074/2500 [01:18<01:34, 15.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1076/2500 [01:18<01:38, 14.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1080/2500 [01:18<01:58, 12.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1085/2500 [01:19<01:18, 18.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1088/2500 [01:19<01:27, 16.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▎     | 1093/2500 [01:19<01:39, 14.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1095/2500 [01:19<01:32, 15.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1098/2500 [01:20<01:33, 15.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1107/2500 [01:20<01:22, 16.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1110/2500 [01:20<01:21, 16.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1115/2500 [01:21<01:46, 12.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1119/2500 [01:21<01:31, 15.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1121/2500 [01:21<01:46, 12.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1128/2500 [01:22<01:35, 14.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1133/2500 [01:22<01:17, 17.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1138/2500 [01:23<01:45, 12.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1144/2500 [01:23<01:20, 16.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1150/2500 [01:23<01:20, 16.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1152/2500 [01:23<01:37, 13.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 1157/2500 [01:24<01:28, 15.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1159/2500 [01:24<01:53, 11.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1164/2500 [01:24<01:42, 13.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1170/2500 [01:25<01:19, 16.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1172/2500 [01:25<01:23, 15.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1177/2500 [01:25<01:44, 12.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1179/2500 [01:25<01:39, 13.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1183/2500 [01:26<01:33, 14.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1186/2500 [01:26<01:19, 16.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1190/2500 [01:26<01:24, 15.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1192/2500 [01:26<01:58, 11.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1194/2500 [01:27<02:13,  9.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1200/2500 [01:27<01:46, 12.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1204/2500 [01:27<01:34, 13.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1206/2500 [01:28<01:28, 14.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1211/2500 [01:28<01:25, 15.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▊     | 1215/2500 [01:28<01:28, 14.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1220/2500 [01:28<00:59, 21.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1225/2500 [01:29<01:15, 16.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1229/2500 [01:29<01:31, 13.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1233/2500 [01:29<01:24, 14.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1237/2500 [01:30<01:40, 12.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1239/2500 [01:30<01:34, 13.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1245/2500 [01:30<01:09, 18.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1249/2500 [01:30<01:23, 14.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1252/2500 [01:30<01:13, 16.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1258/2500 [01:31<01:14, 16.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1261/2500 [01:31<01:22, 14.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1266/2500 [01:31<01:32, 13.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1268/2500 [01:32<01:39, 12.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1270/2500 [01:32<01:57, 10.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1275/2500 [01:32<01:36, 12.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1282/2500 [01:33<01:11, 17.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1288/2500 [01:33<01:25, 14.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1290/2500 [01:33<01:19, 15.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1292/2500 [01:34<01:33, 12.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1294/2500 [01:34<01:44, 11.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1300/2500 [01:34<01:23, 14.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1302/2500 [01:34<01:32, 13.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1306/2500 [01:35<01:24, 14.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1312/2500 [01:35<01:17, 15.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1316/2500 [01:35<01:21, 14.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1320/2500 [01:36<01:36, 12.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1322/2500 [01:36<01:33, 12.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1325/2500 [01:36<01:43, 11.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1331/2500 [01:37<01:19, 14.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1335/2500 [01:37<01:42, 11.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▎    | 1339/2500 [01:37<01:29, 13.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1341/2500 [01:37<01:34, 12.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1347/2500 [01:38<01:26, 13.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1349/2500 [01:38<02:02,  9.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1356/2500 [01:39<01:13, 15.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1361/2500 [01:39<01:06, 17.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1366/2500 [01:39<01:30, 12.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1370/2500 [01:40<01:30, 12.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1372/2500 [01:40<01:46, 10.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1377/2500 [01:40<01:21, 13.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1380/2500 [01:40<01:09, 16.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1382/2500 [01:40<01:08, 16.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1386/2500 [01:41<01:32, 12.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1390/2500 [01:41<01:23, 13.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1392/2500 [01:41<01:28, 12.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1399/2500 [01:42<01:09, 15.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1403/2500 [01:42<01:15, 14.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1405/2500 [01:42<01:31, 12.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1410/2500 [01:43<01:22, 13.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1412/2500 [01:43<01:20, 13.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1414/2500 [01:43<01:37, 11.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1420/2500 [01:43<01:10, 15.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1423/2500 [01:44<01:02, 17.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1425/2500 [01:44<01:15, 14.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1431/2500 [01:44<01:05, 16.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1433/2500 [01:44<01:23, 12.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1437/2500 [01:45<01:17, 13.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1439/2500 [01:45<01:17, 13.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1445/2500 [01:45<01:24, 12.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1447/2500 [01:46<01:35, 10.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1449/2500 [01:46<01:40, 10.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1453/2500 [01:46<01:32, 11.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1460/2500 [01:46<00:59, 17.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1462/2500 [01:47<01:09, 14.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▊    | 1466/2500 [01:47<01:16, 13.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1468/2500 [01:47<01:16, 13.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1474/2500 [01:48<01:13, 13.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1476/2500 [01:48<01:20, 12.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1481/2500 [01:48<01:00, 16.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1484/2500 [01:48<01:02, 16.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1486/2500 [01:48<01:28, 11.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1494/2500 [01:49<01:01, 16.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1499/2500 [01:49<01:06, 14.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1501/2500 [01:49<01:13, 13.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1506/2500 [01:50<01:15, 13.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1508/2500 [01:50<01:12, 13.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1514/2500 [01:50<01:02, 15.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1516/2500 [01:51<01:08, 14.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1521/2500 [01:51<01:03, 15.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1525/2500 [01:51<00:51, 18.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1530/2500 [01:51<01:08, 14.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1533/2500 [01:52<01:13, 13.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1535/2500 [01:52<01:13, 13.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1538/2500 [01:52<01:00, 15.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1543/2500 [01:52<01:07, 14.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1546/2500 [01:53<00:55, 17.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1551/2500 [01:53<01:21, 11.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1554/2500 [01:53<01:13, 12.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1557/2500 [01:53<01:06, 14.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1562/2500 [01:54<01:18, 11.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1566/2500 [01:54<00:57, 16.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1569/2500 [01:54<01:05, 14.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1574/2500 [01:55<01:01, 15.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1581/2500 [01:55<00:49, 18.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1584/2500 [01:55<01:01, 14.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1586/2500 [01:56<01:23, 10.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1592/2500 [01:56<01:11, 12.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1594/2500 [01:56<01:28, 10.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1597/2500 [01:57<01:09, 12.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1606/2500 [01:57<00:48, 18.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1609/2500 [01:57<01:18, 11.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1611/2500 [01:58<01:31,  9.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1620/2500 [01:58<00:54, 16.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1625/2500 [01:59<01:14, 11.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1631/2500 [01:59<01:02, 13.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1634/2500 [01:59<01:01, 14.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1642/2500 [02:00<00:47, 18.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1645/2500 [02:00<01:03, 13.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1649/2500 [02:00<01:07, 12.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1654/2500 [02:01<00:51, 16.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1658/2500 [02:01<00:52, 15.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1662/2500 [02:01<01:13, 11.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1664/2500 [02:02<01:12, 11.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1668/2500 [02:02<01:10, 11.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1675/2500 [02:02<00:44, 18.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1677/2500 [02:03<01:14, 11.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1680/2500 [02:03<00:59, 13.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1684/2500 [02:03<01:01, 13.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1691/2500 [02:03<00:57, 14.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1693/2500 [02:04<01:01, 13.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1697/2500 [02:04<01:07, 11.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1700/2500 [02:04<00:53, 15.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1707/2500 [02:04<00:40, 19.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1710/2500 [02:05<00:57, 13.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 1716/2500 [02:05<00:45, 17.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1719/2500 [02:06<01:03, 12.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 1726/2500 [02:06<00:54, 14.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1729/2500 [02:06<00:54, 14.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1731/2500 [02:07<01:14, 10.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 1738/2500 [02:07<01:02, 12.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 1741/2500 [02:07<00:51, 14.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 1744/2500 [02:08<00:52, 14.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 1749/2500 [02:08<00:51, 14.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1752/2500 [02:08<00:44, 16.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1755/2500 [02:09<01:12, 10.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1757/2500 [02:09<01:09, 10.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1764/2500 [02:09<00:57, 12.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1766/2500 [02:09<00:57, 12.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1770/2500 [02:10<00:54, 13.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1772/2500 [02:10<00:58, 12.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1776/2500 [02:10<00:52, 13.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1778/2500 [02:10<00:47, 15.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 1783/2500 [02:11<00:47, 15.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1788/2500 [02:11<00:48, 14.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1793/2500 [02:11<00:32, 21.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1796/2500 [02:11<00:47, 14.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1800/2500 [02:12<00:42, 16.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1804/2500 [02:12<00:47, 14.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1809/2500 [02:12<00:48, 14.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1813/2500 [02:13<00:49, 13.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1819/2500 [02:13<00:55, 12.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1822/2500 [02:13<00:49, 13.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1826/2500 [02:14<00:43, 15.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1831/2500 [02:14<00:50, 13.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1836/2500 [02:14<00:39, 17.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▎  | 1841/2500 [02:15<00:34, 18.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1844/2500 [02:15<00:53, 12.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1849/2500 [02:15<00:52, 12.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1854/2500 [02:16<00:52, 12.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1858/2500 [02:16<00:47, 13.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1860/2500 [02:16<00:52, 12.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1862/2500 [02:17<00:57, 11.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 1869/2500 [02:17<00:43, 14.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 1873/2500 [02:17<00:43, 14.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1875/2500 [02:17<00:50, 12.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 1879/2500 [02:18<00:50, 12.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1882/2500 [02:18<00:46, 13.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 1887/2500 [02:19<00:55, 10.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1893/2500 [02:19<00:40, 14.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1896/2500 [02:19<00:42, 14.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1900/2500 [02:20<00:53, 11.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1903/2500 [02:20<00:47, 12.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▋  | 1908/2500 [02:20<00:50, 11.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1916/2500 [02:21<00:41, 14.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1921/2500 [02:21<00:47, 12.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1925/2500 [02:22<00:45, 12.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1928/2500 [02:22<00:45, 12.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1930/2500 [02:22<00:50, 11.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1935/2500 [02:22<00:48, 11.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1940/2500 [02:23<00:36, 15.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1942/2500 [02:23<00:42, 13.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1946/2500 [02:23<00:42, 13.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1948/2500 [02:23<00:40, 13.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1953/2500 [02:24<00:37, 14.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1958/2500 [02:24<00:35, 15.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1960/2500 [02:24<00:40, 13.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 1966/2500 [02:25<00:44, 12.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 1968/2500 [02:25<00:40, 13.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1970/2500 [02:25<00:46, 11.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1974/2500 [02:25<00:41, 12.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1979/2500 [02:26<00:36, 14.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1983/2500 [02:26<00:35, 14.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 1989/2500 [02:26<00:33, 15.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 1994/2500 [02:27<00:31, 16.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 1997/2500 [02:27<00:26, 19.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2002/2500 [02:27<00:38, 13.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2006/2500 [02:28<00:34, 14.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2008/2500 [02:28<00:33, 14.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2010/2500 [02:28<00:39, 12.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2016/2500 [02:28<00:33, 14.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2018/2500 [02:29<00:35, 13.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2022/2500 [02:29<00:36, 13.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2024/2500 [02:29<00:37, 12.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2029/2500 [02:29<00:29, 15.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2033/2500 [02:30<00:33, 14.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2035/2500 [02:30<00:36, 12.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2044/2500 [02:30<00:21, 21.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2049/2500 [02:31<00:34, 12.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2053/2500 [02:31<00:31, 14.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2055/2500 [02:31<00:35, 12.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2060/2500 [02:32<00:33, 13.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2065/2500 [02:32<00:28, 15.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2068/2500 [02:32<00:23, 18.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2071/2500 [02:32<00:37, 11.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2078/2500 [02:33<00:25, 16.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2083/2500 [02:33<00:30, 13.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2087/2500 [02:33<00:27, 15.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2089/2500 [02:34<00:31, 13.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2093/2500 [02:34<00:31, 12.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2097/2500 [02:34<00:28, 14.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2102/2500 [02:35<00:31, 12.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2107/2500 [02:35<00:30, 13.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2109/2500 [02:35<00:31, 12.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2111/2500 [02:35<00:33, 11.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2115/2500 [02:36<00:26, 14.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2120/2500 [02:36<00:28, 13.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2122/2500 [02:36<00:26, 14.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2127/2500 [02:37<00:26, 14.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2131/2500 [02:37<00:33, 11.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2141/2500 [02:38<00:30, 11.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2143/2500 [02:38<00:36,  9.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2146/2500 [02:38<00:29, 11.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2152/2500 [02:39<00:22, 15.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2155/2500 [02:39<00:21, 16.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2158/2500 [02:39<00:27, 12.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2165/2500 [02:40<00:23, 14.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2167/2500 [02:40<00:30, 11.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2176/2500 [02:41<00:21, 14.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2178/2500 [02:41<00:22, 14.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2183/2500 [02:41<00:20, 15.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2185/2500 [02:41<00:20, 15.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2189/2500 [02:41<00:21, 14.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2191/2500 [02:42<00:23, 13.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2197/2500 [02:42<00:20, 14.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2203/2500 [02:42<00:19, 15.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2205/2500 [02:43<00:20, 14.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2209/2500 [02:43<00:16, 17.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 2213/2500 [02:43<00:20, 13.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2215/2500 [02:43<00:22, 12.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2221/2500 [02:44<00:19, 14.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2223/2500 [02:44<00:19, 14.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2227/2500 [02:44<00:24, 11.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2230/2500 [02:44<00:18, 14.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2234/2500 [02:45<00:19, 13.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2238/2500 [02:45<00:20, 13.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2242/2500 [02:45<00:17, 14.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2244/2500 [02:46<00:18, 13.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2247/2500 [02:46<00:17, 14.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2253/2500 [02:46<00:15, 16.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2255/2500 [02:46<00:19, 12.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2259/2500 [02:47<00:17, 13.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2264/2500 [02:47<00:14, 16.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2269/2500 [02:47<00:19, 12.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2272/2500 [02:48<00:17, 13.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2276/2500 [02:48<00:18, 12.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2278/2500 [02:48<00:20, 10.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2286/2500 [02:48<00:10, 19.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2289/2500 [02:49<00:12, 16.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2292/2500 [02:49<00:12, 16.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2295/2500 [02:49<00:13, 15.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2299/2500 [02:49<00:14, 13.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2302/2500 [02:50<00:15, 12.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2307/2500 [02:50<00:13, 14.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2309/2500 [02:50<00:12, 15.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2312/2500 [02:50<00:13, 13.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2314/2500 [02:51<00:19,  9.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2319/2500 [02:51<00:14, 12.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2324/2500 [02:51<00:10, 16.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2328/2500 [02:52<00:12, 13.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2333/2500 [02:52<00:11, 15.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2336/2500 [02:52<00:11, 14.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2340/2500 [02:52<00:09, 16.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2342/2500 [02:53<00:11, 13.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2345/2500 [02:53<00:10, 14.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2347/2500 [02:53<00:12, 12.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2351/2500 [02:53<00:11, 13.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2355/2500 [02:54<00:10, 13.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2362/2500 [02:54<00:08, 16.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2365/2500 [02:54<00:10, 13.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2371/2500 [02:55<00:08, 15.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2373/2500 [02:55<00:08, 14.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2377/2500 [02:55<00:07, 15.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2382/2500 [02:56<00:08, 13.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2384/2500 [02:56<00:08, 13.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2386/2500 [02:56<00:09, 11.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2389/2500 [02:56<00:08, 12.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2393/2500 [02:56<00:07, 14.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2399/2500 [02:57<00:05, 17.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2402/2500 [02:57<00:07, 12.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▋| 2407/2500 [02:57<00:06, 14.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▋| 2411/2500 [02:58<00:06, 14.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2413/2500 [02:58<00:06, 12.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2417/2500 [02:58<00:05, 14.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2422/2500 [02:58<00:04, 16.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2425/2500 [02:59<00:04, 16.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2427/2500 [02:59<00:06, 11.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2433/2500 [02:59<00:04, 13.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2435/2500 [03:00<00:05, 11.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2438/2500 [03:00<00:04, 13.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2443/2500 [03:00<00:04, 13.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2445/2500 [03:00<00:04, 11.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2450/2500 [03:01<00:03, 13.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2455/2500 [03:01<00:03, 12.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2459/2500 [03:01<00:03, 13.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2461/2500 [03:02<00:02, 14.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▊| 2467/2500 [03:02<00:02, 14.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2469/2500 [03:02<00:02, 13.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2471/2500 [03:02<00:02, 11.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2475/2500 [03:03<00:02, 11.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2479/2500 [03:03<00:01, 13.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2482/2500 [03:03<00:01, 15.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2484/2500 [03:03<00:01, 12.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2490/2500 [03:04<00:00, 12.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2492/2500 [03:04<00:00, 11.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2494/2500 [03:04<00:00, 12.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2496/2500 [03:05<00:00,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2499/2500 [03:05<00:00,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|██████████| 2500/2500 [03:06<00:00, 13.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-27 17:55:07.157\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m13\u001b[0m - \u001b[1mEvaluation metrics (after optimization): {'f1': 0.0, 'em': 0.0, 'acc': 0.7108}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "logger.info(\"Evaluating workflow on test set...\")\n",
    "with suppress_logger_info():\n",
    "    results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "logger.info(f\"Evaluation metrics (before optimization): {results}\")\n",
    "\n",
    "logger.info(\"Optimizing workflow...\")\n",
    "textgrad_optimizer.optimize(benchmark, seed=8)\n",
    "textgrad_optimizer.restore_best_graph()\n",
    "\n",
    "logger.info(\"Evaluating workflow on test set...\")\n",
    "with suppress_logger_info():\n",
    "    results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "logger.info(f\"Evaluation metrics (after optimization): {results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4b6f274d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'f1': 0.0, 'em': 0.0, 'acc': 0.7108}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a4af1fcb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 18:29:35.583\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.workflow.workflow_graph\u001b[0m:\u001b[36msave_module\u001b[0m:\u001b[36m1201\u001b[0m - \u001b[1mSaving SequentialWorkFlowGraph to ./debug/textgradinfo.json\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "textgrad_optimizer.save(\"./debug/textgradinfo.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b463a4a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from evoagentx.agents.agent_manager import AgentManager\n",
    "from evoagentx.benchmark import HotPotQA\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.core.logging import logger\n",
    "from evoagentx.evaluators import Evaluator\n",
    "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
    "from evoagentx.optimizers import TextGradOptimizer\n",
    "from evoagentx.prompts import StringTemplate\n",
    "from evoagentx.workflow import SequentialWorkFlowGraph\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from evoagentx.agents.agent_manager import AgentManager\n",
    "from evoagentx.benchmark import MBPP\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.core.logging import logger\n",
    "from evoagentx.evaluators import Evaluator\n",
    "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
    "from evoagentx.optimizers import TextGradOptimizer\n",
    "from evoagentx.prompts import StringTemplate\n",
    "from evoagentx.workflow import SequentialWorkFlowGraph\n",
    "\n",
    "from evoagentx.models import OpenAILLMConfig, OpenAILLM\n",
    "from evoagentx.workflow import SEWWorkFlowGraph, STRUCTUREWorkFlowGraph\n",
    "from evoagentx.agents import AgentManager\n",
    "from evoagentx.benchmark import HumanEval,AFlowMBPP\n",
    "from evoagentx.evaluators import Evaluator \n",
    "from evoagentx.optimizers import SEWOptimizer, STRUCTUREOptimizer\n",
    "from evoagentx.optimizers.structure_optimizer import STRUCTUREWorkFlowScheme\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "\n",
    "from evoagentx.models import OpenAILLMConfig, OpenAILLM,AzureOpenAIConfig,LiteLLMConfig,LiteLLM\n",
    "from evoagentx.workflow import SEWWorkFlowGraph \n",
    "from evoagentx.agents import AgentManager\n",
    "from evoagentx.benchmark import MBPPPLUS, AFlowMBPPPLUS\n",
    "from evoagentx.evaluators import Evaluator \n",
    "from evoagentx.optimizers import SEWOptimizer \n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.benchmark import HumanEvalPLUS\n",
    "from evoagentx.benchmark import SciCode\n",
    "from copy import deepcopy\n",
    "\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "480de58c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class HotPotQASplits(HotPotQA):\n",
    "\n",
    "    def _load_data(self):\n",
    "        # load the original test data \n",
    "        super()._load_data()\n",
    "        # split the data into train, dev and test\n",
    "        import numpy as np \n",
    "        np.random.seed(42)\n",
    "        permutation = np.random.permutation(len(self._dev_data))\n",
    "        full_test_data = self._dev_data \n",
    "        # randomly select 10 samples for train, 40 for dev, and 100 for test\n",
    "        self._train_data = [full_test_data[idx] for idx in permutation[:50]]\n",
    "        self._dev_data = [full_test_data[idx] for idx in permutation[:50]]\n",
    "        self._test_data = [full_test_data[idx] for idx in permutation[50:550]]\n",
    "        self._fulldata = full_test_data\n",
    "\n",
    "\n",
    "def collate_func(example: dict) -> dict:\n",
    "    context_list = []\n",
    "    for item in example[\"context\"]:\n",
    "        context = \"Title: {}\\nText: {}\".format(item[0], \" \".join([t.strip() for t in item[1]]))\n",
    "        context_list.append(context)\n",
    "    context = \"\\n\\n\".join(context_list)\n",
    "    problem = \"Context: {}\\n\\nQuestion: {}\\n\\nAnswer:\".format(context, example[\"question\"])\n",
    "    return {\"problem\": problem}\n",
    "\n",
    "\n",
    "hotpotqa_graph_data = {\n",
    "    \"goal\": \"Answer the question based on the context. The answer should be a direct response to the question, without including explanations or reasoning.\",\n",
    "    \"tasks\": [\n",
    "        {\n",
    "            \"name\": \"answer_generate\",\n",
    "            \"description\": \"Answer the question based on the context.\",\n",
    "            \"inputs\": [\n",
    "                {\"name\": \"problem\", \"type\": \"str\", \"required\": True, \"description\": \"The problem to solve.\"}\n",
    "            ],\n",
    "            \"outputs\": [\n",
    "                {\"name\": \"answer\", \"type\": \"str\", \"required\": True, \"description\": \"The answer to the problem.\"}\n",
    "            ],\n",
    "            \"prompt_template\": StringTemplate(instruction=\"Think step by step to answer the question. You should explain your thinking process in the 'thought' field, and provide the final answer in the 'answer' field.\\nFormat your output in xml format, such as <thought>xxx</thought> and <answer>xxx</answer>.\"),\n",
    "            \"parse_mode\": \"xml\"\n",
    "        }\n",
    "    ] \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e9818857",
   "metadata": {},
   "outputs": [],
   "source": [
    "# os.environ[\"AZURE_OPENAI_DEPLOYMENT_NAME\"] = \"gpt-4o-mini\"\n",
    "# os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"https://optimizehumaneval.cognitiveservices.azure.com/\"\n",
    "# os.environ[\"AZURE_OPENAI_KEY\"] = \"2b7h6anDXRsl5XHDUAGKHpjh3DLv9kLjcjGXN6PvsEmLVf1i3imMJQQJ99BKACYeBjFXJ3w3AAABACOGATqP\"\n",
    "# os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"2025-01-01-preview\"\n",
    "# llm_config = LiteLLMConfig(model=\"azure/\" + os.getenv(\"AZURE_OPENAI_DEPLOYMENT_NAME\"),  # Azure model format\n",
    "#     azure_endpoint=os.getenv(\"AZURE_OPENAI_ENDPOINT\"),\n",
    "#     azure_key=os.getenv(\"AZURE_OPENAI_KEY\"),\n",
    "#     api_version=os.getenv(\"AZURE_OPENAI_API_VERSION\", \"2024-12-01-preview\"), top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
    "\n",
    "# executor_llm = LiteLLM(config=llm_config)\n",
    "# optimizer_llm = LiteLLM(config=llm_config)\n",
    "\n",
    "os.environ[\"AZURE_OPENAI_DEPLOYMENT_NAME\"] = \"gpt-4o-mini\"\n",
    "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"https://75244-mfztkr7x-eastus2.cognitiveservices.azure.com/\"\n",
    "os.environ[\"AZURE_OPENAI_KEY\"] = \"8PNMdsUYGdMPsCfl0baO0hjtnGE2m40zJTrUGC3vKnHdpjnkOgeQJQQJ99BIACHYHv6XJ3w3AAAAACOG7VZI\"\n",
    "os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"2025-01-01-preview\"\n",
    "llm_config = LiteLLMConfig(model=\"azure/\" + os.getenv(\"AZURE_OPENAI_DEPLOYMENT_NAME\"),  # Azure model format\n",
    "    azure_endpoint=os.getenv(\"AZURE_OPENAI_ENDPOINT\"),\n",
    "    azure_key=os.getenv(\"AZURE_OPENAI_KEY\"),\n",
    "    api_version=os.getenv(\"AZURE_OPENAI_API_VERSION\", \"2024-12-01-preview\"), top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
    "\n",
    "executor_llm = LiteLLM(config=llm_config)\n",
    "optimizer_llm = LiteLLM(config=llm_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "796739dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 12:45:41.759\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.hotpotqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m51\u001b[0m - \u001b[1mloading HotPotQA data from /gpfs/radev/home/tl688/.evoagentx/data/hotpotqa/hotpot_train_v1.1.json ...\u001b[0m\n",
      "\u001b[32m2025-12-09 12:45:45.836\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.hotpotqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m51\u001b[0m - \u001b[1mloading HotPotQA data from /gpfs/radev/home/tl688/.evoagentx/data/hotpotqa/hotpot_dev_distractor_v1.json ...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "benchmark = HotPotQASplits()\n",
    "workflow_graph = SequentialWorkFlowGraph.from_dict(hotpotqa_graph_data)\n",
    "agent_manager = AgentManager()\n",
    "agent_manager.add_agents_from_workflow(workflow_graph, executor_llm.config)\n",
    "\n",
    "evaluator = Evaluator(\n",
    "    llm=executor_llm, \n",
    "    agent_manager=agent_manager, \n",
    "    collate_func=collate_func, \n",
    "    num_workers=20, \n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "textgrad_optimizer = TextGradOptimizer(\n",
    "    graph=workflow_graph, \n",
    "    optimize_mode=\"all\",\n",
    "    executor_llm=executor_llm, \n",
    "    optimizer_llm=optimizer_llm,\n",
    "    batch_size=3,\n",
    "    max_steps=20,\n",
    "    evaluator=evaluator,\n",
    "    eval_every_n_steps=1,\n",
    "    eval_rounds=1,\n",
    "    save_interval=None,\n",
    "    save_path=\"./\",\n",
    "    rollback=True,\n",
    "    constraints=[]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "baa44bb7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7405"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(benchmark._fulldata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3ed1f571",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-07 19:30:03.447\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m1\u001b[0m - \u001b[1mEvaluating workflow on test set...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 1/500 [00:01<12:47,  1.54s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 6/500 [00:01<01:41,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 10/500 [00:02<01:07,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 12/500 [00:02<00:54,  8.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 16/500 [00:02<00:39, 12.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 19/500 [00:02<00:43, 11.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 23/500 [00:03<00:52,  9.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 25/500 [00:03<01:02,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7272727272727273, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 29/500 [00:04<00:41, 11.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 32/500 [00:04<00:38, 12.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 34/500 [00:04<00:50,  9.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4827586206896552, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 37/500 [00:04<00:49,  9.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 39/500 [00:05<00:50,  9.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.13333333333333333, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▊         | 43/500 [00:05<00:46,  9.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 45/500 [00:05<00:47,  9.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 48/500 [00:05<00:38, 11.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 50/500 [00:06<00:53,  8.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 52/500 [00:06<00:49,  9.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 57/500 [00:06<00:40, 10.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 59/500 [00:07<00:43, 10.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.08695652173913045, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 61/500 [00:07<00:49,  8.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 63/500 [00:07<00:49,  8.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 65/500 [00:07<00:50,  8.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 68/500 [00:08<00:45,  9.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 72/500 [00:08<00:45,  9.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.30769230769230765, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.28571428571428575, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 78/500 [00:08<00:24, 17.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 81/500 [01:00<34:30,  4.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 85/500 [01:01<20:14,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 91/500 [01:01<09:20,  1.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.1111111111111111, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 94/500 [01:01<06:30,  1.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 97/500 [01:01<04:32,  1.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 100/500 [01:02<03:50,  1.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 102/500 [01:03<03:16,  2.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 108/500 [01:03<01:39,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.28571428571428575, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 110/500 [01:03<01:26,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4615384615384615, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 112/500 [01:03<01:12,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.9090909090909091, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 114/500 [01:04<01:09,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.16666666666666669, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 116/500 [01:04<01:03,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4799999999999999, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 120/500 [01:04<00:45,  8.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 123/500 [01:05<00:39,  9.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 127/500 [01:05<00:44,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 130/500 [01:06<00:47,  7.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 132/500 [01:06<00:38,  9.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 136/500 [01:06<00:36, 10.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5185185185185185, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 141/500 [01:06<00:22, 15.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 143/500 [01:07<00:30, 11.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 145/500 [01:07<00:31, 11.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444444, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 147/500 [01:07<00:33, 10.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 149/500 [01:08<00:47,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.21052631578947367, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 154/500 [01:08<00:38,  9.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 156/500 [01:08<00:35,  9.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 161/500 [01:08<00:27, 12.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.42857142857142855, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 164/500 [01:09<00:31, 10.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 166/500 [01:09<00:35,  9.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.17391304347826084, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 169/500 [01:10<00:39,  8.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 171/500 [01:10<00:49,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 172/500 [01:10<00:50,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7142857142857143, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 173/500 [02:01<1:08:26, 12.56s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.13333333333333333, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 179/500 [02:01<17:33,  3.28s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.15384615384615383, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 184/500 [02:01<08:09,  1.55s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 187/500 [02:01<05:24,  1.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.047619047619047616, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 189/500 [02:02<04:37,  1.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 191/500 [02:03<03:35,  1.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 196/500 [02:03<01:50,  2.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 199/500 [02:03<01:19,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 203/500 [02:04<00:59,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.09090909090909091, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 208/500 [02:04<00:38,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 210/500 [02:05<00:41,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 215/500 [02:05<00:26, 10.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.7777777777777778, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 217/500 [02:05<00:24, 11.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 219/500 [02:05<00:27, 10.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 221/500 [02:06<00:34,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 227/500 [02:06<00:25, 10.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 229/500 [02:06<00:26, 10.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.21428571428571425, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.15384615384615385, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 231/500 [02:07<00:31,  8.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 233/500 [02:07<00:29,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 239/500 [02:07<00:22, 11.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3636363636363636, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 244/500 [02:08<00:24, 10.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 246/500 [02:08<00:25,  9.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 248/500 [02:08<00:26,  9.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 251/500 [02:09<00:30,  8.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 253/500 [02:09<00:29,  8.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 257/500 [02:09<00:26,  9.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 261/500 [02:10<00:19, 12.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 263/500 [02:10<00:17, 13.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 265/500 [02:10<00:28,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 267/500 [02:11<00:46,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.061224489795918366, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 269/500 [03:01<25:32,  6.64s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 271/500 [03:01<16:36,  4.35s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 280/500 [03:01<04:16,  1.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5625, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.11764705882352941, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 283/500 [03:02<03:03,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 286/500 [03:03<02:32,  1.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 290/500 [03:03<01:37,  2.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 295/500 [03:03<00:51,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2222222222222222, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 297/500 [03:04<00:46,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 301/500 [03:04<00:35,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 304/500 [03:05<00:31,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.9090909090909091, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 306/500 [03:05<00:30,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 308/500 [03:05<00:23,  8.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 310/500 [03:05<00:24,  7.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3571428571428571, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 315/500 [03:06<00:18, 10.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 319/500 [03:06<00:23,  7.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 322/500 [03:07<00:17, 10.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 327/500 [03:07<00:12, 13.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 329/500 [03:07<00:11, 14.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 331/500 [03:07<00:16, 10.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 335/500 [03:08<00:20,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 337/500 [03:08<00:18,  8.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 339/500 [03:08<00:18,  8.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.27118644067796605, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 343/500 [03:09<00:15,  9.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3157894736842105, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 347/500 [03:09<00:18,  8.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.14285714285714285, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 352/500 [03:10<00:14, 10.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.048780487804878044, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 354/500 [03:10<00:16,  8.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 358/500 [03:10<00:14, 10.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 360/500 [03:11<00:15,  9.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.21052631578947367, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 361/500 [04:01<19:51,  8.57s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 365/500 [04:01<08:58,  3.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 371/500 [04:02<03:12,  1.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.05263157894736842, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 373/500 [04:02<02:28,  1.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 375/500 [04:02<01:55,  1.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6363636363636364, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 378/500 [04:03<01:14,  1.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 379/500 [04:03<01:03,  1.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 384/500 [04:03<00:26,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 386/500 [04:03<00:20,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 389/500 [04:04<00:18,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4347826086956522, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 391/500 [04:05<00:22,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 394/500 [04:05<00:19,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 398/500 [04:05<00:12,  8.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 400/500 [04:05<00:10,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 402/500 [04:06<00:11,  8.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 404/500 [04:06<00:09,  9.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.24000000000000002, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 406/500 [04:06<00:09,  9.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 409/500 [04:06<00:11,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 413/500 [04:07<00:08,  9.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.09523809523809525, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 420/500 [04:07<00:05, 14.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 422/500 [04:08<00:07, 11.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 424/500 [04:08<00:08,  8.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 426/500 [04:08<00:09,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 431/500 [04:09<00:07,  9.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2666666666666667, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 433/500 [04:09<00:07,  9.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 436/500 [04:09<00:05, 10.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4210526315789474, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 440/500 [04:10<00:05, 10.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 442/500 [04:10<00:05, 10.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 444/500 [04:10<00:06,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 446/500 [04:10<00:05,  9.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8235294117647058, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 451/500 [04:11<00:04, 11.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 455/500 [05:01<03:45,  5.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 457/500 [05:01<02:35,  3.61s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 462/500 [05:01<01:08,  1.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5263157894736842, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 466/500 [05:02<00:38,  1.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 471/500 [05:03<00:16,  1.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 473/500 [05:03<00:12,  2.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.47619047619047616, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 475/500 [05:03<00:10,  2.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 477/500 [05:03<00:07,  3.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 479/500 [05:04<00:05,  3.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.06060606060606061, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▋| 482/500 [05:04<00:03,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 483/500 [05:05<00:04,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 486/500 [05:05<00:02,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 488/500 [05:05<00:01,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.375, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 490/500 [05:05<00:01,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 494/500 [05:06<00:00, 10.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 496/500 [05:06<00:00, 10.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 498/500 [05:07<00:00,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 499/500 [05:07<00:00,  3.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.28571428571428575, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 500/500 [05:08<00:00,  1.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.1, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2025-12-07 19:35:11.871\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m4\u001b[0m - \u001b[1mEvaluation metrics (before optimization): {'f1': 0.6032480280405156, 'em': 0.428, 'acc': 0.632}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "logger.info(\"Evaluating workflow on test set...\")\n",
    "with suppress_logger_info():\n",
    "    results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "logger.info(f\"Evaluation metrics (before optimization): {results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0f33f493",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'f1': 0.6032480280405156, 'em': 0.428, 'acc': 0.632}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ceebbf97",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "46206f6d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "500"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(textgrad_optimizer.evaluator._evaluation_records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "713d6040",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 1/500 [00:01<14:52,  1.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 2/500 [00:02<07:50,  1.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 4/500 [00:02<03:34,  2.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 7/500 [00:02<01:30,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 11/500 [00:03<01:02,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 13/500 [00:03<00:51,  9.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 15/500 [00:03<01:04,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 18/500 [00:04<01:26,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 20/500 [00:04<01:07,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7272727272727273, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 25/500 [00:05<01:27,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 26/500 [00:05<01:22,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 29/500 [00:06<01:12,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 31/500 [00:06<01:19,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 32/500 [00:06<01:32,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 35/500 [00:07<01:13,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 37/500 [00:07<01:11,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 38/500 [00:07<01:42,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 40/500 [00:08<01:46,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.13333333333333333, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 45/500 [00:08<00:50,  8.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 47/500 [00:09<00:59,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 49/500 [00:09<01:23,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 55/500 [00:10<01:19,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 56/500 [00:10<01:15,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 59/500 [00:11<01:05,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.125, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 63/500 [00:11<00:47,  9.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 65/500 [00:12<01:07,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 66/500 [00:12<01:18,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 68/500 [00:12<01:11,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 69/500 [00:12<01:20,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 72/500 [00:13<01:11,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 74/500 [00:13<01:03,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 75/500 [00:13<01:02,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 78/500 [00:14<01:09,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 80/500 [00:14<01:07,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 85/500 [00:15<00:45,  9.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.30769230769230765, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.23529411764705882, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 87/500 [00:15<00:54,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.16666666666666669, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-09 13:03:55.422\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac1276c554299294b2190bc', 'answer': 'Euthanasia Program', 'question': \"Heinrich Gross participated in the Nazi regime's Aktion T4 more commonly referred to as what?\", 'supporting_facts': [['Heinrich Gross', 0], ['Aktion T4', 3]], 'context': [['Karl Freiherr Michel von Tüßling', ['Karl Freiherr Michel von Tüßling (27 July 1907 – 30 October 1991) was a Schutzstaffel (SS) officer who served in the Nazi government of German dictator Adolf Hitler and in the SS Main Office.', ' From 1936 onwards, he was the personal adjutant of \"Reichsleiter\" and SS-\"Obergruppenführer\" Philipp Bouhler, who was in charge of Hitler\\'s Chancellery (Kanzlei des Führers), head of the euthanasia programme Aktion T4, as well as co-initiator of Aktion 14f13.', ' In 1947 Tüßling provided an affidavit in defence of war criminal Viktor Brack who was sentenced to death at the Nuremberg trials.']], ['Heinrich Gross', ['Heinrich Gross (14 November 1915 – 15 December 2005) was an Austrian psychiatrist, medical doctor and neurologist, a reputed expert as a leading court-appointed psychiatrist, ill-famed for his proven involvement in the killing of at least nine children with physical, mental and/or emotional/behavioral characteristics considered \"unclean\" by the Nazi regime, under its Euthanasia Program.', ' His role in hundreds of other cases of infanticide is unclear.', \" Gross was head of the Spiegelgrund children's psychiatric clinic for two years during World War II.\"]], ['Am Spiegelgrund clinic', [\"Am Spiegelgrund was the name of a children's clinic in Vienna where 789 children were killed under the Nazi Regime Children's Euthanasia Program, also known as Aktion T4.\", ' Between 1940-1945, the clinic operated as part of the psychological institution “Am Steinhof” (renamed the Otto Wagner Clinic) on the Baumgartner Höhe, now located in Penzing, the 14th district of Vienna.', ' This clinic was divided into a reform school and a sanatorium for children, which included a so-called Children’s Ward, where sick, disabled, and otherwise ‘un-educable’ adolescents were abused and subjected to harsh medical experiments.', \" Some died by lethal injection and gas poisoning; others by disease, undernourishment, exposure to the elements, and 'accidents' relating to their conditions.\", ' The brains of up to 800 victims were preserved in jars and housed in the hospital for decades.']], ['Philipp Bouhler', ['Philipp Bouhler (11 September 1899 – 19 May 1945) was a senior Nazi Party official who was both a \"Reichsleiter\" (National Leader) and Chief of the Chancellery of the Führer of the NSDAP.', ' He was also an SS-\"Obergruppenführer\" in the \"Allgemeine SS\" who was responsible for the Nazi \"Aktion T4\" euthanasia program that killed more than 70,000 handicapped adults and children in Nazi Germany, as well as co-initiator of \"Aktion 14f13\", also called \"Sonderbehandlung\" (\"special treatment\"), that killed 15,000–20,000 concentration camp prisoners.']], ['Artur Hojan', ['Artur Hojan (7 August 1973 – found dead, 9 February 2014) was a journalist and published author specializing in the history of the Chełmno extermination camp and the Nazi involuntary euthanasia programme conducted in the territory of occupied Poland by the SS during World War\\xa0II.', ' Hojan was the co-founder of the \"Tiergartenstrasse4\" Association in 2005 (together with Cameron Munro) devoted to Aktion T4 history, with emphasis on the Kościan psychiatric hospital located where he lived.', ' Hojan, age of 40, left home in the evening of 1\\xa0December 2013 at 8\\xa0p.m. for a walk around town and disappeared.', ' His body was found two months later on 9\\xa0February 2014 floating in the Obra canal near the town of Kiełczewo, and identified later.', ' The cause of death has not been determined.', ' He was buried at the Kościan cemetery on 15\\xa0February 2014.', ' He left behind a wife and young daughter.', ' The monograph \"Treblinka Death Camp: History, Biographies, Remembrance\" by Chris Webb, the co-founder of H.E.A.R.T (also known as the HolocaustResearchProject.org), is dedicated to his memory.']], ['Aktion T4', ['Aktion T4 (German, ] ) was a postwar name for mass murder through involuntary euthanasia in Nazi Germany.', ' The name T4 is an abbreviation of \"Tiergartenstraße 4\", a street address of the Chancellery department set up in the spring of 1940, in the Berlin borough of Tiergarten, which recruited and paid personnel associated with T4.', ' Certain German physicians were authorized to select patients \"deemed incurably sick, after most critical medical examination\" and then administer to them a \"mercy death\" (\"Gnadentod\") .', ' In October 1939 Adolf Hitler signed a \"euthanasia decree\" backdated to 1 September 1939 that authorized his personal physician Karl Brandt and \"Reichsleiter\" Philipp Bouhler to implement the programme.']], ['Karl Brandt', ['Karl Brandt (January 8, 1904 – June 2, 1948) was a German physician and \"Schutzstaffel\" (SS) officer in Nazi Germany.', \" Trained in surgery, Brandt joined the Nazi Party in 1932 and became Adolf Hitler's escort physician in August 1934.\", ' A member of Hitler\\'s inner circle at the Berghof, he was selected by Philipp Bouhler, the head of Hitler\\'s Chancellery, to administer the \"Aktion T4\" euthanasia program.', ' Brandt was later appointed the Reich Commissioner of Sanitation and Health (\"Bevollmächtigter für das Sanitäts- und Gesundheitswesen\").', ' Accused of involvement in human experimentation and other war crimes, Brandt was indicted in late 1946 and faced trial before a U.S. military tribunal along with 22 others in \"United States of America v. Karl Brandt, et al\".', ' He was convicted, sentenced to death, and later hanged on June 2, 1948.']], ['Gerhard Kretschmar', ['Gerhard Herbert Kretschmar (20 February 1939 – 25 July 1939), was a German child born with severe disabilities.', \" After receiving a petition from the child's parents, the German Führer Adolf Hitler authorized one of his personal physicians, Karl Brandt, to have the child killed.\", ' This marked the beginning of the program in Nazi Germany known as a \"euthanasia program\" (Aktion T4) which ultimately resulted in the deliberate killing of about 200,000 people with mental and/or physical disabilities.']], ['Memorandum Authorizing Involuntary Euthanasia', ['Adolf Hitler signed a memorandum authorizing involuntary euthanasia in October 1939 to serve as the legal basis for Aktion T4, the Nazi forced euthanasia program.', ' Its purpose was to assure the doctors and nurses who took part in the euthanasia program would not be prosecuted for murder.', ' During the postwar trials of these same individuals, they attempted to use this decree as a justification for their actions.']], ['Dasein ohne Leben', ['Dasein ohne Leben – Psychiatrie und Menschlichkeit (\"Existence Without Life\" – \"Psychiatry and Humanity\") is a 1942 Nazi propaganda film about the physically and mentally disabled: closeups of disabled persons.', ' The director was Hermann Schwenninger, one of the three managing directors of Gemeinnützige Krankentransport (\"Charitable Ambulance\"), a front company of Aktion T4, the central institution for the mass murder of patients in the Third Reich.', ' Schwenninger also wrote parts of the screenplay of \"Ich klage an\".', \" The contract for the film came from Hitler's Chancellery, and was produced by Tobis Film.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The input to LLMOutputParser.parse should be a str, but found <class 'NoneType'>.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 90/500 [00:16<01:11,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 92/500 [00:16<01:17,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 93/500 [00:17<01:45,  3.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 95/500 [00:17<01:18,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 96/500 [00:17<01:54,  3.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.27272727272727276, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 99/500 [00:18<01:14,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 101/500 [00:18<01:06,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.1739130434782609, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 103/500 [00:18<00:59,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.1111111111111111, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 104/500 [00:18<01:05,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 106/500 [00:19<01:00,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.9090909090909091, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 109/500 [00:19<01:17,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 110/500 [00:20<01:11,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4615384615384615, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 112/500 [00:20<01:02,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 113/500 [00:20<01:13,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 117/500 [00:21<00:53,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 118/500 [00:21<01:05,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 120/500 [00:21<01:07,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.028169014084507043, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 123/500 [00:22<00:53,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 125/500 [00:22<00:44,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.7777777777777778, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 129/500 [00:23<00:56,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 130/500 [00:23<00:57,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 131/500 [00:23<01:23,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 132/500 [00:23<01:22,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 133/500 [00:24<01:25,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 135/500 [00:24<01:04,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 136/500 [00:24<01:22,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 144/500 [00:25<00:30, 11.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 146/500 [00:25<00:29, 12.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 148/500 [00:26<01:00,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 152/500 [00:26<00:52,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.42857142857142855, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 155/500 [00:27<01:20,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 157/500 [00:27<01:01,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 159/500 [00:28<01:04,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 162/500 [00:28<00:47,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 166/500 [00:29<00:53,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 170/500 [00:29<00:39,  8.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.625, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 175/500 [00:30<00:47,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 177/500 [00:31<01:11,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.1081081081081081, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 180/500 [00:31<00:52,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 183/500 [00:32<00:52,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 185/500 [00:32<00:43,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 186/500 [00:32<00:45,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 188/500 [00:32<00:44,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 190/500 [00:33<00:49,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.08333333333333334, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 193/500 [00:34<01:00,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.28571428571428575, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 194/500 [00:34<00:56,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 196/500 [00:34<01:00,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 200/500 [00:35<00:43,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 204/500 [00:35<00:38,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444444, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 206/500 [00:36<01:00,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7777777777777778, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 207/500 [00:36<01:03,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 210/500 [00:37<00:48,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 213/500 [00:37<00:52,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 215/500 [00:38<00:51,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 217/500 [00:38<00:40,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 219/500 [00:38<00:45,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 220/500 [00:39<00:47,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 222/500 [00:39<00:41,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.21428571428571425, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 225/500 [00:39<00:38,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.15384615384615385, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 226/500 [00:39<00:44,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 228/500 [00:40<00:49,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 230/500 [00:40<00:58,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 232/500 [00:41<00:51,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 234/500 [00:41<00:40,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 236/500 [00:41<00:37,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 238/500 [00:42<00:49,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 242/500 [00:42<00:34,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 245/500 [00:43<00:40,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.14285714285714285, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 246/500 [00:43<00:42,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 250/500 [00:43<00:34,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 252/500 [00:44<00:32,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 254/500 [00:44<00:49,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.13333333333333336, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 255/500 [00:45<00:55,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 257/500 [00:45<00:55,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 260/500 [00:46<00:50,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 263/500 [00:46<00:32,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.09302325581395349, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 264/500 [00:46<00:34,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 266/500 [00:46<00:32,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 267/500 [00:47<00:46,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 270/500 [00:47<00:37,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 272/500 [00:47<00:36,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 273/500 [00:48<00:41,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 276/500 [00:48<00:33,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5625, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 277/500 [00:48<00:36,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 279/500 [00:48<00:32,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 282/500 [00:49<00:28,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 284/500 [00:49<00:24,  8.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 287/500 [00:50<00:33,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 289/500 [00:50<00:45,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.23529411764705882, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 292/500 [00:51<00:32,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 293/500 [00:51<00:37,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 295/500 [00:51<00:32,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 296/500 [00:51<00:42,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 299/500 [00:52<00:42,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.625, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 302/500 [00:52<00:29,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 305/500 [00:53<00:29,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 306/500 [00:53<00:34,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 310/500 [00:54<00:34,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3571428571428571, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.34285714285714286, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 313/500 [00:54<00:29,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 316/500 [00:55<00:19,  9.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 320/500 [00:55<00:23,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 322/500 [00:56<00:43,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 325/500 [00:57<00:33,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 328/500 [00:57<00:27,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 332/500 [00:58<00:24,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3157894736842105, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 335/500 [00:58<00:25,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 336/500 [00:58<00:29,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 340/500 [00:59<00:22,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 341/500 [00:59<00:26,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 343/500 [00:59<00:23,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.23529411764705882, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 346/500 [01:00<00:25,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 349/500 [01:01<00:26,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 351/500 [01:01<00:33,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 352/500 [01:01<00:31,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 356/500 [01:02<00:20,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 358/500 [01:02<00:21,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 359/500 [01:02<00:19,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 361/500 [01:03<00:28,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 362/500 [01:03<00:29,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 364/500 [01:04<00:26,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.048780487804878044, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 365/500 [01:04<00:29,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 367/500 [01:04<00:24,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 368/500 [01:04<00:27,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 370/500 [01:05<00:23,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 374/500 [01:05<00:14,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 375/500 [01:05<00:22,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.125, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 377/500 [01:06<00:21,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 379/500 [01:06<00:20,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 380/500 [01:06<00:22,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.06451612903225806, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 384/500 [01:07<00:17,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 385/500 [01:07<00:22,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 388/500 [01:08<00:17,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 389/500 [01:08<00:21,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 391/500 [01:08<00:17,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 392/500 [01:09<00:24,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 395/500 [01:09<00:18,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 396/500 [01:09<00:20,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 400/500 [01:10<00:17,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 403/500 [01:10<00:12,  7.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6363636363636364, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 405/500 [01:10<00:12,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.41379310344827586, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534590a72d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff3e00>, 9882831.900972921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534590a75d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ee2490>\n",
      "Evaluating workflow:  81%|████████  | 406/500 [01:11<00:19,  4.88it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f2cd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c9fd0>, 9882838.278584408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f2cc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345904a8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c9240>, 9882832.884850688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ee2810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff1ef0>, 9882831.427523242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345904a790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f132d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cb0e0>, 9882833.9227552)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534595cd9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f75bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458fc6cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590caf20>, 9882833.485776952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f759d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc06e0>, 9882833.419836726)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458fc7410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458fb3b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459302f90>, 9882833.734559556)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458fb3a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534590838d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1cc0>, 9882836.545759743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d08ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458cfe6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c3d0>, 9882838.642686889)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458cff3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ed0f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458cf7810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458cfe750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d40e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c8c90>, 9882835.569353769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ed1590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1940>, 9882835.45715636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458cf7d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff2d60>, 9882835.484050402)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458cfe610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb2900>, 9882835.978691092)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d40c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d70e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb2580>, 9882836.362148808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d70c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458dce210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9e9e0>, 9882839.550886922)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458dce390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ea1790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9f0e0>, 9882843.748531394)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ea1b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c7fad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9ff50>, 9882839.236639148)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c7fd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d23f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c66fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c7d390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ca8990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9fcb0>, 9882839.608670356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c7d250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9c520>, 9882839.61855446)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c9b190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9d1d0>, 9882838.926174395)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d23bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9f770>, 9882839.920571996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c66c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b47ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9d9b0>, 9882842.418300232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b47dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b2fe10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9cd00>, 9882845.62442739)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c7e110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534589fd050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff19b0>, 9882844.463479863)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589fd250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a33d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9d940>, 9882844.491282685)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458db1c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b77c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ba2450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458bcc050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534589d8ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a27ad0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b782f0>, 9882842.748177644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b77b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7b230>, 9882842.367578967)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a275d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3930>, 9882841.798520438)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ba2310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b79240>, 9882842.813740004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458bcc4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b79c50>, 9882842.62341245)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589d9210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a6f950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a710f0>, 9882844.99469701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a45550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a96a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b79cc0>, 9882846.026654517)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a96b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534589f1050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896de10>, 9882846.658695927)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589f1290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b50c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458903d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458925d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a731c0>, 9882845.31032316)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458925c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a71e10>, 9882844.703264192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b510d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a7b0>, 9882844.08250844)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458903c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c85f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534588609f0>, 9882848.343938269)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589620d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b77590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a722e0>, 9882848.521123527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b77b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534589610d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534588b17d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a350>, 9882847.712289272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458960d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9da90>, 9882847.97851446)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534588b1410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458864950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458860670>, 9882848.871845668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458864750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587b3d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745fd0>, 9882852.710383805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587b3c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587c9e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747d90>, 9882851.40044568)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587cb190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345868d410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896ec80>, 9882852.974455036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345868d610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458673290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b79550>, 9882854.090643408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458673d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587972d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585db3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745860>, 9882852.663472926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458794f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458862270>, 9882851.131142477)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585d9110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585ae8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536190>, 9882856.114402125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585ae850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534586cbc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536cf0>, 9882855.69037116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534586cbb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585fa810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863cf30>, 9882854.83743494)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585f9550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534584fd3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863f2a0>, 9882854.435514037)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458506dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345865b450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458434210>, 9882857.476258684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345865b4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583ecb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458436660>, 9882856.70534608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534583ecbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585a0650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458200ad0>, 9882861.13784732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458215690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585be2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201630>, 9882859.93803291)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534582398d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585fa650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583ac5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534584aee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458744280>, 9882860.098073509)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458728550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534583169e0>, 9882859.607659576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534583ac450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435f60>, 9882859.847287528)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581dc310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534584aed10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863c4b0>, 9882864.1207297)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584af6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d4b590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f3700>, 9882866.442518309)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581b4250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345815ca10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f1da0>, 9882873.879695881)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345815ca50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534581a29d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314280>, 9882864.060176007)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581a2fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458136910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458316270>, 9882862.26817764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581367d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f8f050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f60290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458233cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f1a270>, 9882865.285610208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f8ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f19a20>, 9882865.184962856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f60550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863f700>, 9882866.705135787)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458233f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345912cad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459154950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345916ebd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459171250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534591c9390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345913c110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458feb9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534591b0310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534591b2250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ff7750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459015550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459081610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534591ca390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459185090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534590d22d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459322d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458efee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458fda690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345904ab50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f64990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f90c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f1f590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458dedf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e03110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e18850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345904a850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e59310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e75e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534590a6890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e90dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e27b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d21b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d92850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458dc02d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ed2f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d89f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458bee3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c118d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c1a2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d93cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c9a250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458cb2e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b08390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b187d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b44f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b5d8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458bfd810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b5f110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458addb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a0fbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a4d610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a5f090>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a868d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a9d810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ab5410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345894d9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458960a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345898c210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534589a4dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534589c5d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587e6450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534588011d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345891c9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345881a950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534588676d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345888e990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534586e75d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534586f42d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458702090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587fb610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345897be10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534586e4390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587f8ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345877dd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534588b9cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585ed1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585eef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534586115d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458750ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345864eed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345865ac90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458699410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534586af790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534586c2d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585ee290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458522a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345854cad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345857d7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458505350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585bdbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583ef550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458591fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345841cf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345857c050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583e7750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458423190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458442f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345857ed10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534584c6410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534584cf510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534582fe8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583191d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583319d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345854c710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345834f6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458498850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458479d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345837a1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345838d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583c00d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583d1bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458233810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345826cb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345826f410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458295890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458299f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534582a8d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458256310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534580e9750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458118f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534581296d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345837a5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458191550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534581a9e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f15190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534581caed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f49590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458166450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458181450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345826f6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345983fe70>, 9882830.367798489)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534592ccfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f1550>, 9882831.371525154)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459155c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590da820>, 9882829.125914356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345916ea90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f0f30>, 9882830.484667726)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459171150>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590dbcb0>, 9882829.811011951)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534591c9190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590dadd0>, 9882830.677291391)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345913c5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff0520>, 9882831.22430714)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458feb450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590da0b0>, 9882829.879768025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534591b0810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590db700>, 9882829.891834762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534592c2990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff07c0>, 9882836.544556811)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ff5e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff1550>, 9882833.004508631)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534590178d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534593010f0>, 9882833.306392608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459080750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff30e0>, 9882833.31641138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534591ca4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590db4d0>, 9882833.027719624)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534591851d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c84b0>, 9882832.133219423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534590c0b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c8d00>, 9882832.533216286)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459322e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff0a60>, 9882832.69662018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534592cee90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c89f0>, 9882833.009495305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458fdac10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff3310>, 9882836.978533447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345904a910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cadd0>, 9882832.867393028)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f65050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cb700>, 9882834.822920892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459061250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cba10>, 9882836.11979208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f1fd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc1010>, 9882834.238347083)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458dedbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc1630>, 9882852.547752276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459293610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff1b70>, 9882836.390956368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e1b950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f3c40>, 9882838.008158552)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534590488d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c8670>, 9882835.396068484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e58e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3620>, 9882837.240396252)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e75c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3f50>, 9882836.76737256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f2d6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3d90>, 9882836.702975493)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e90c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff00c0>, 9882836.604469024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f28a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459302c10>, 9882839.639974916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d21910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb08a0>, 9882838.504123325)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d92650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3460>, 9882838.289261768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458dc0790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb0f30>, 9882858.547463996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ed3190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0590>, 9882842.918526353)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d89d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f0520>, 9882838.132578876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458bee290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cb310>, 9882839.134384302)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459081d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d8d00>, 9882839.7768469)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c1a190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c830>, 9882841.637364028)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d93d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9d6a0>, 9882840.540892938)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458cd3190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9dc50>, 9882841.25154344)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e1b190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9de10>, 9882841.157439565)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458af11d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9e7b0>, 9882840.521569893)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b1ad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c9e80>, 9882842.516898043)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b45650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9f9a0>, 9882843.18295312)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b5e010>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9e820>, 9882840.614111474)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534591cbb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9fd90>, 9882841.54491216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b5f9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9f620>, 9882845.939949809)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458addc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7aba0>, 9882842.735971356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a0fa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7bb60>, 9882844.893009888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a4e050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7bf50>, 9882845.763164388)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a5eb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a70830>, 9882844.389407497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a86e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a70de0>, 9882845.63025631)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a9d6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a71a90>, 9882846.544608936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458dcd1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a73af0>, 9882846.367566925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345894dd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896c2f0>, 9882845.806240644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458953190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896cb40>, 9882848.017825903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345898c8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896d2b0>, 9882848.164142264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589a4bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a740>, 9882847.961253632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589c5c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896e040>, 9882850.184232617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587e6b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896e970>, 9882846.966827348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458801a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a73000>, 9882846.695345648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345891d290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590caac0>, 9882848.281656545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345881a810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534588604b0>, 9882850.301262196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534588674d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a72ac0>, 9882850.081019595)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345888ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a71780>, 9882850.340481183)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534586e7d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b78fa0>, 9882848.89417808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458962050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458863150>, 9882849.067896636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458701f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a73b60>, 9882849.348733516)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458803290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896c830>, 9882850.123670006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589a6090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896d7f0>, 9882849.458319815)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534586dffd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458744fa0>, 9882849.976969851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587f8b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534587451d0>, 9882852.970568301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345877db10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745be0>, 9882852.756527748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587e6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458862660>, 9882851.524925016)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585ecd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745b70>, 9882851.36258266)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585ef1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534587475b0>, 9882852.208805691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534586113d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534587477e0>, 9882852.38886576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458750fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458746f90>, 9882856.552408598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345864ed90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863cd70>, 9882852.6346298)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345865ab50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863da90>, 9882852.173985152)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458961450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863e2e0>, 9882855.17999348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534586af350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863ec10>, 9882853.290288351)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534586c0e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863f620>, 9882855.752354112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585ed250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458534050>, 9882855.69870137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458522690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585348a0>, 9882856.025856001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345854cfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458535a20>, 9882854.846533272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345857de90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745400>, 9882855.398519088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585045d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536740>, 9882858.665255833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585be290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585374d0>, 9882855.564027146)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534583ef410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585379a0>, 9882855.741857)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458592bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458537b60>, 9882857.849587863)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345841cd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747230>, 9882855.066393744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345857c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863ee40>, 9882855.74778936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534583e7610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458434050>, 9882856.518196588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458422dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458434c90>, 9882860.676346233)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458442e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584343d0>, 9882857.350065542)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345857ea10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534587466d0>, 9882860.801763875)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584c6790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437310>, 9882857.86699012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584cf310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747cb0>, 9882857.401718596)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534582fee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458860d70>, 9882860.62367409)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458319890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458315010>, 9882859.515757697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458331810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458315240>, 9882858.031064808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584c7e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458315400>, 9882858.30883852)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345834f4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584364a0>, 9882857.737940295)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d7fdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314600>, 9882857.783762977)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345847a490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435da0>, 9882860.98114276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345837a050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534583165f0>, 9882860.617244668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345838d6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458537ee0>, 9882860.646632558)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f67350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437ee0>, 9882862.10548674)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534583d1c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201010>, 9882861.217488589)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458231850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201e10>, 9882861.914757792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345826c910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582000c0>, 9882862.581165327)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458fa7d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458202740>, 9882861.39330765)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458295690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582029e0>, 9882886.055100188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458332550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201470>, 9882863.62616032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534582a8c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201be0>, 9882862.03502244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345848d550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458203e70>, 9882865.116673896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534580e9550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458317e00>, 9882867.651596362)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458118d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458434b40>, 9882866.101532647)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581294d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f1860>, 9882864.3882452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345837a2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458202430>, 9882862.831469087)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458191410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458316660>, 9882865.626735162)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458272450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f18210>, 9882863.662269572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f14f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f0de0>, 9882865.35721851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581caf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f19240>, 9882867.602738757)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f49450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f02f0>, 9882863.739758171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458166a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f27b0>, 9882864.336323071)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458181410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201e80>, 9882864.186263744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345826f4d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345900af90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458803590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f71f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459300130>, 9882830.630989872)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345900b690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896f460>, 9882848.748650262)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458803990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f19cc0>, 9882864.882146614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f719d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a4ef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ba9810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458bc47d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534589f0450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b2e410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458fd3e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534592b7190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345977b1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534595183d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345958bf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459722c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459a08690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459ae1f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e580>, 9883849.362887615)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fac65a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8f460>, 9883849.46449386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153460351c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8dfd0>, 9883849.77909316)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587ebb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8ce50>, 9883849.852011088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458bc58d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8f770>, 9883850.01806002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589f0d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8c6e0>, 9883850.533831237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b2f950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e120>, 9883851.22707024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458fd3510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e430>, 9883851.42781599)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534592b71d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8d240>, 9883849.066391263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345977b090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8d080>, 9883851.30401605)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459518b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d74360>, 9883848.712876832)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459588250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8cc90>, 9883854.070222097)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534598d2290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8da90>, 9883849.952562304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459a09fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8fb60>, 9883849.829261934)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459b713d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a0800d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a0c89d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a1d1dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a09cb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534596c4a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8d5c0>, 9883850.284753729)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a09ea50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a013210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e5f0>, 9883849.571961029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459e57b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8eac0>, 9883850.077623246)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a0c8510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8f070>, 9883849.539761344)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a1d1710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8fbd0>, 9883849.481346363)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534596c4710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8fd90>, 9883850.46309058)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459f43810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345841e510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b58440>, 9883851.363755915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a2fff90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459277850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d77770>, 9883851.123304488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a46850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534590c2f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a330e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b589f0>, 9883855.602370575)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589f06d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2510>, 9883851.381485393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a330710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a323b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a568d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0360>, 9883852.987619191)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a320c90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e6d0>, 9883851.42422691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a599ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a550650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1630>, 9883853.400218781)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a551010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a0e1350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a471250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0210>, 9883852.615392197)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a0e0890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2200>, 9883852.64354076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a59bf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459b721d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a42df10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3d20>, 9883853.04456058)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459ae20d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a41a490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a5c2650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345983d0f0>, 9883852.355189033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a42e790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345aabab30>, 9883852.696989477)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a41add0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1780>, 9883853.946893793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a5c3b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a7c6490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a6be010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae31c0>, 9883854.358604085)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a7c6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8d710>, 9883854.847490994)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a6bea10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a629810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3310>, 9883855.26945972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a62a390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a52ddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5cfa0>, 9883857.946523728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a09ec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a672450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a797990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5c1a0>, 9883854.128044246)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a670750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5c980>, 9883855.390472023)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458af1650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a780410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f930>, 9883869.053328088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459218fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a95f550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5c0c0>, 9883855.465034852)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534596c6e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a722b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a853190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5eeb0>, 9883855.47299862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a720b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a8bd710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5edd0>, 9883855.461567327)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a8bd410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345ad77610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a7da610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5e200>, 9883859.026595611)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345ad74390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5dc50>, 9883856.054972436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a7da250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a838050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a6a7f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458fa4cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a2c2090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d630>, 9883855.389149357)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a6dc4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f460>, 9883855.820167545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a362790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae2cf0>, 9883852.479217464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a6a7650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8db1c0>, 9883853.833369948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f3ff50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3cb0>, 9883853.73377564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459518290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a4f7c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8f310>, 9883856.670646897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a4f76d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a918c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae1a510>, 9883857.868799925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a918310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a9f3c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19a90>, 9883856.67239898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a9f37d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a928dd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19240>, 9883856.659241436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a928210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534596a0750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fadd9e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2c10>, 9883858.512184376)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534596a2590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae2eb0>, 9883856.63370036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fadd8f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345ad10750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f2a0>, 9883857.623085132)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345ad0fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345aa00250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030b930>, 9883858.221275045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a702050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a966f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a9bc650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15358ea0c9f0>, 9883859.931994876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a647d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941e820>, 9883858.097406248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fadd5c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a9be110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fad7e3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941e270>, 9883861.281571547)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fade45d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fada4a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941e430>, 9883858.535930185)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fada6fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533faffd150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a896d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae1b5b0>, 9883855.983257772)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a5c3190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941fe00>, 9883857.751291564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fad7e310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941eeb0>, 9883858.456012012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533faffed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345f58bb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941fa80>, 9883857.594234912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345f58b8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d6708d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae18210>, 9883859.590555202)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15346034b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a9e2890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d68abd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941e040>, 9883861.914793031)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a9e2bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e2e0>, 9883859.357506672)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d68a990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d665250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d99b0>, 9883859.318509392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d6663d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fad76d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d8c20>, 9883860.541691393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fad76910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d6ae890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f1160>, 9883860.68340071)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d6ad4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345f66ea50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f0ec0>, 9883859.869623508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345f66fe90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d701f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d525ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d763c0>, 9883860.109149503)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d702090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f3e00>, 9883862.456284696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d526a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a52f210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d53fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f12b0>, 9883860.432861332)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a52f310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f08a0>, 9883860.287306588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d53fa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d55d210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345f66edd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d652690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a70fa0>, 9883861.091053547)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d55cf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030af90>, 9883858.276078012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345f66ec50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5de10>, 9883858.322472775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d652350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d581810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d91d0>, 9883862.98103558)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d581650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d593ed0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff10f0>, 9883861.992317837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d593c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d5ac350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f3b60>, 9883861.109991845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d5ac790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a9b1450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941e190>, 9883863.152682845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d6c8bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d5d7790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513540>, 9883861.920519987)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d5d7b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d6be350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d60f810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459511860>, 9883862.06981597)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d700b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d430c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534595134d0>, 9883863.392150424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a8314d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d43c910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae1a5f0>, 9883868.891983386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d43c550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d4433d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d467e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b580c0>, 9883862.374583762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a52e810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962ee40>, 9883864.049968235)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d467c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d47fed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962f700>, 9883862.71506101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d47f990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d4cc590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a734150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d56eed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d5c2290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513a10>, 9883877.495650845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d60f610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459512c80>, 9883861.346780324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d5c20d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941f700>, 9883865.565493697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d4cd810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941f690>, 9883861.924181644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a7341d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff0750>, 9883861.596849415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d56ecd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d456950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962f8c0>, 9883864.882207826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a942210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d4f7ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c0c0>, 9883864.02880185)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d4f7dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d32e610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962ff50>, 9883865.344861267)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d32ed50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d35f650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962d860>, 9883864.8364415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d60fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d5685d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459512cf0>, 9883868.545652373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d568250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3782d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973b9a0>, 9883865.43022344)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d378790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d4a0190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d38a890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973aba0>, 9883865.13001909)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345ad311d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973ae40>, 9883864.821268056)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d38a690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d399c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a430>, 9883865.081122562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d399ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3aca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534597399b0>, 9883866.062277464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a463a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d390a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3cb050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973ba80>, 9883865.810902536)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d390ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3ce6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973b310>, 9883866.978464944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3cee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3e2ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459eb2010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d36ac10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff3930>, 9883865.7958848)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3cb550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962d710>, 9883867.226814639)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3e2fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c280>, 9883863.579615075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459eb39d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c7c0>, 9883864.249071704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d36aa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d46cf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff3770>, 9883866.985894512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d5d7cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d417c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962fe00>, 9883867.568500508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d414550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d41df50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993ca60>, 9883868.92898061)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d41de10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbdac90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993d0f0>, 9883871.074654723)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbda8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d40b010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbef610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c0c0>, 9883867.525371438)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d81e810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbf6ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993cec0>, 9883867.884729648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbf6e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3ee790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc0b5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962d1d0>, 9883869.25146002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d379ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a38280>, 9883867.856202101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc0b490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc15e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3ac80>, 9883867.985611025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc16510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc17fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc267d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c3d0>, 9883868.679785144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc17b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a6d0>, 9883868.28774234)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc26e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3ec410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbc7650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff1be0>, 9883869.12661705)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbef410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d406050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534597387c0>, 9883866.100529375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3ee250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993fb60>, 9883866.703035831)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbc7450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513e00>, 9883866.851448992)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d431c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc315d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a970>, 9883877.660464885)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc31c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d40fa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc7ba10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3bc40>, 9883869.821326602)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d40ff50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3be00>, 9883870.6010231)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc7b8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc96a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3acf0>, 9883870.122139636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc962d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbfba90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738a60>, 9883871.668116655)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbfb190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bcae210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bcbaed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5e6d0>, 9883870.820193816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bcae0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534595109f0>, 9883871.982261393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bcbacd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443120510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993e6d0>, 9883871.130505145)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344311af90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443120410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5e200>, 9883871.957915325)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443121190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344312da50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d860>, 9883872.054017004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443109c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344312f250>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443151490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993d710>, 9883871.732824553)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344312d650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5d8d0>, 9883872.048699256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443151b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443110a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5dfd0>, 9883872.016599173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344315b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbc74d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc68a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738d70>, 9883869.211284088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3e37d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3bbd0>, 9883869.245605491)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc69110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc8db90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993f700>, 9883869.95929484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc8de90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459511f60>, 9883871.9490929)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443110b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443170150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431ba910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5e0b0>, 9883873.074752325)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443171410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19be70>, 9883873.532437539)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534431ba7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431d1410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993d780>, 9883873.042375982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534431d1210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431ea010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a199780>, 9883874.923016954)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534431ea650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443012310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443118390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443029f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5cb40>, 9883873.438914677)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443118650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0927b0>, 9883873.616519678)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443029dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443181a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc9d750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3bcb0>, 9883875.759797977)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534431babd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a093af0>, 9883874.823078737)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc9d2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344318e050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443077fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443176090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443171d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344318e010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431a5450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a39be0>, 9883873.198037554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443012110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a199240>, 9883872.092353657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344319ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19a6d0>, 9883872.211928232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbc7e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a090b40>, 9883874.115875892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344318ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5ed60>, 9883876.285346184)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443077ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a38d70>, 9883871.69424784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443176790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19ac80>, 9883871.279823)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344318e710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443085d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5e3c0>, 9883876.405022824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430853d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344303a910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0911d0>, 9883875.336004755)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344303b6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f03150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a38bb0>, 9883876.530942973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f034d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443055510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f1ca10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993c590>, 9883876.400675919)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443056bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e74910>, 9883876.250240894)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430d4fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534430f6cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e746e0>, 9883875.609879144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430f7d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534430e2c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f46610>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e748a0>, 9883876.719585368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430e2d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5ec10>, 9883876.8704342)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f46d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f4ed10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e76e40>, 9883876.5928175)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f4ebd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f86c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f9b9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344307fdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344309b810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534430b2250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443016190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534430d67d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0935b0>, 9883877.463587523)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f87350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a198130>, 9883875.494053496)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344309ae10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a092c10>, 9883875.234184232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443016210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d1d0>, 9883879.239422165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f9b890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a093770>, 9883874.66167274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344307fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a091fd0>, 9883875.471925803)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430b2110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a090210>, 9883874.792627366)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430d6e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534430b2550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e77000>, 9883880.86973719)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430b23d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431e2dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e74a60>, 9883877.525021683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d5aee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f8dd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442fb4350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a198750>, 9883878.542953927)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442fca810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b1080>, 9883880.220225845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442fd7f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442fef290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0937e0>, 9883878.593561426)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442fef810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442ff11d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e09d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0750>, 9883879.035588907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ff13d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b2f90>, 9883878.406462269)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e0a590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e24890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0908a0>, 9883879.546239076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e24690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e38990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f1c7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0590>, 9883881.05845917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f68050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b3310>, 9883878.518842245)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f1d250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3d2d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e67f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b1630>, 9883878.859043049)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443087690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b1160>, 9883878.716016956)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e67e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431e1250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e82ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b38c0>, 9883879.214836383)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e71c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0c20>, 9883879.57089372)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e82cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e94e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3b150>, 9883879.370571708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e94c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442ead5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c9400>, 9883879.842866253)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442eac290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442ebfb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442ed9610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c99b0>, 9883882.650525456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ebff90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c9f60>, 9883881.186949164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ed9cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442ee4690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a092ba0>, 9883881.643986708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ee47d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bcb4d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442ff0410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b2900>, 9883881.224946825)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344309b890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ca900>, 9883881.980106764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ff04d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442eeecd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d3efd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c9160>, 9883884.732017431)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442eef950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ca510>, 9883882.579265784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d3f6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d4ab90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cb930>, 9883882.583317945)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d4aa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d22350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d74910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b17f0>, 9883880.960535683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d21350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b5940>, 9883881.967822172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442fa1bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d802d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d9b890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b63c0>, 9883882.065750545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d81f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7310>, 9883885.723241812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d9b690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442db2bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b6200>, 9883883.734724348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442db3310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e95050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442dc71d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442ebdf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b79a0>, 9883882.12856295)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e95d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b6040>, 9883885.684897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442dc4850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8d70>, 9883879.823369976)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ebe650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442dd2090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442de5cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b5710>, 9883884.029810842)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442de5910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d77f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cbd20>, 9883884.834357588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443047590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c48910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e76c10>, 9883883.480035037)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c48750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c12790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c57290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b15c0>, 9883888.425742673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c12f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c9940>, 9883884.11027442)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c57150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c95990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e2660>, 9883885.31586289)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c95850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442de9810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442cb5d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e1be0>, 9883885.183701837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442deab90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e0050>, 9883885.100548504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442cb6410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c1ef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442cd10d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e1470>, 9883885.24619112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c1e8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a092c80>, 9883887.19604382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442cd15d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442cf5b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f3a10>, 9883887.64605254)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442cf53d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c85e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442910910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b04b0>, 9883883.959761718)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d47890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e64610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c03550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a092660>, 9883882.69869134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c03350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2cf0>, 9883884.728115184)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ec8210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b2c10>, 9883886.199227097)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442910710>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3540>, 9883883.380188229)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c26550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344291d9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344293a6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f0590>, 9883886.154766804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344291e110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f0f30>, 9883886.375482732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344293add0>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.13333333333333333, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 410/500 [01:12<00:17,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 414/500 [01:12<00:12,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 416/500 [01:12<00:13,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 421/500 [01:13<00:10,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3157894736842105, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 423/500 [01:13<00:09,  8.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 424/500 [01:14<00:15,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 427/500 [01:14<00:11,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 429/500 [01:14<00:11,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 431/500 [01:15<00:11,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 435/500 [01:15<00:08,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 436/500 [01:15<00:08,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 441/500 [01:17<00:10,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8235294117647058, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 443/500 [01:17<00:08,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 445/500 [01:17<00:07,  7.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.0909090909090909, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 447/500 [01:17<00:06,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 449/500 [01:17<00:07,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 453/500 [01:18<00:08,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 456/500 [01:19<00:06,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5263157894736842, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 460/500 [01:20<00:07,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 462/500 [01:20<00:05,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 467/500 [01:20<00:03, 10.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 469/500 [01:20<00:02, 11.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 471/500 [01:21<00:05,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 473/500 [01:22<00:04,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.47619047619047616, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 475/500 [01:22<00:05,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 479/500 [01:23<00:03,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 481/500 [01:23<00:02,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 483/500 [01:23<00:02,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 484/500 [01:23<00:02,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 485/500 [01:24<00:03,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 487/500 [01:24<00:03,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.375, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 489/500 [01:25<00:01,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 490/500 [01:25<00:01,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 494/500 [01:25<00:00,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 495/500 [01:26<00:00,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 498/500 [01:26<00:00,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 499/500 [01:27<00:00,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 500/500 [01:30<00:00,  5.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.044444444444444446, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 3/500 [00:02<04:42,  1.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 5/500 [00:02<02:38,  3.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 8/500 [00:02<01:33,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 11/500 [00:02<01:24,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 15/500 [00:03<01:05,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 16/500 [00:03<01:11,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 18/500 [00:04<01:44,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 21/500 [00:04<01:11,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 22/500 [00:04<01:23,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7272727272727273, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 23/500 [00:04<01:26,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 29/500 [00:05<00:53,  8.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 31/500 [00:06<01:10,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 33/500 [00:06<01:21,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 34/500 [00:06<01:24,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 35/500 [00:07<01:38,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4827586206896552, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 37/500 [00:07<01:34,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 38/500 [00:07<01:38,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 40/500 [00:08<01:44,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 42/500 [00:08<01:36,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 43/500 [00:08<01:31,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 46/500 [00:09<01:10,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 50/500 [00:09<00:43, 10.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.13333333333333333, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 55/500 [00:09<00:45,  9.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 58/500 [00:11<01:41,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 63/500 [00:12<01:15,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.125, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 68/500 [00:12<01:02,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 70/500 [00:13<00:57,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 73/500 [00:14<01:27,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 74/500 [00:14<01:21,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 77/500 [00:14<01:07,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 79/500 [00:14<00:55,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 81/500 [00:14<00:50,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 83/500 [00:15<01:03,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 85/500 [00:15<01:10,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.16666666666666669, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 86/500 [00:16<01:12,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 87/500 [00:16<01:38,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 91/500 [00:16<00:58,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 92/500 [00:17<01:39,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 93/500 [00:17<01:36,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 100/500 [00:18<01:02,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.28571428571428575, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 102/500 [00:18<00:59,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3636363636363636, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 104/500 [00:19<00:56,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 106/500 [00:19<01:01,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 108/500 [00:19<00:50,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 110/500 [00:20<00:51,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.9090909090909091, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4615384615384615, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 112/500 [00:20<01:12,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4615384615384615, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 114/500 [00:20<00:59,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 115/500 [00:21<01:04,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 120/500 [00:21<00:52,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 122/500 [00:21<00:47,  7.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 124/500 [00:22<00:44,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 128/500 [00:22<00:44,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.7777777777777778, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 130/500 [00:23<00:52,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 131/500 [00:23<01:16,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.05, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 133/500 [00:24<01:16,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 134/500 [00:24<01:30,  4.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 137/500 [00:24<01:02,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 142/500 [00:25<00:37,  9.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 144/500 [00:25<00:51,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.21052631578947367, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 148/500 [00:25<00:37,  9.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.7692307692307693, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 150/500 [00:26<01:21,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 152/500 [00:27<01:40,  3.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 156/500 [00:28<01:04,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 158/500 [00:28<00:50,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 160/500 [00:28<00:46,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 162/500 [00:28<00:58,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 164/500 [00:29<00:53,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 166/500 [00:29<00:53,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 168/500 [00:29<00:53,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 169/500 [00:30<00:56,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 171/500 [00:30<00:48,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 176/500 [00:31<00:42,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7142857142857143, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.22222222222222224, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 178/500 [00:32<01:11,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 184/500 [00:32<00:38,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.23076923076923078, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 189/500 [00:33<00:43,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 191/500 [00:33<00:36,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 193/500 [00:33<00:38,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 195/500 [00:34<01:04,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 197/500 [00:34<00:55,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 199/500 [00:35<00:53,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.38095238095238093, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 202/500 [00:35<00:38,  7.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 204/500 [00:35<00:51,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 206/500 [00:36<00:50,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7777777777777778, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 208/500 [00:36<00:57,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 210/500 [00:37<00:47,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 212/500 [00:37<01:19,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 214/500 [00:38<01:20,  3.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 216/500 [00:38<00:53,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 219/500 [00:39<00:40,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 222/500 [00:39<00:32,  8.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.21428571428571425, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 226/500 [00:39<00:23, 11.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.15384615384615385, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 228/500 [00:39<00:30,  8.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 230/500 [00:40<01:01,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 233/500 [00:41<00:51,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 236/500 [00:41<00:43,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 240/500 [00:42<00:34,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▊     | 243/500 [00:42<00:40,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 245/500 [00:43<00:51,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 248/500 [00:43<00:33,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.14285714285714285, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 252/500 [00:44<00:30,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 255/500 [00:44<00:21, 11.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.11764705882352941, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 257/500 [00:44<00:39,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.08333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 261/500 [00:45<00:32,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 263/500 [00:46<00:50,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 266/500 [00:47<00:57,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 271/500 [00:47<00:29,  7.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 274/500 [00:48<00:34,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 276/500 [00:48<00:31,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 277/500 [00:48<00:44,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 279/500 [00:49<00:42,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 280/500 [00:49<00:42,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 282/500 [00:49<00:36,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 285/500 [00:49<00:28,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 288/500 [00:50<00:31,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 289/500 [00:50<00:47,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 291/500 [00:51<00:49,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 292/500 [00:51<00:49,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.625, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 294/500 [00:52<00:55,  3.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 295/500 [00:52<00:59,  3.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 297/500 [00:52<00:44,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 298/500 [00:53<00:58,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 300/500 [00:54<01:06,  3.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 304/500 [00:54<00:41,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 305/500 [00:55<00:46,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3571428571428571, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 307/500 [00:55<00:46,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 308/500 [00:55<00:39,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 311/500 [00:56<00:43,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 314/500 [00:56<00:33,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 316/500 [00:57<00:44,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 318/500 [00:57<00:31,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 322/500 [00:58<00:20,  8.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 325/500 [00:58<00:16, 10.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4615384615384615, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 327/500 [00:58<00:16, 10.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2222222222222222, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 329/500 [00:58<00:18,  9.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 332/500 [00:59<00:34,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 336/500 [01:00<00:22,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 339/500 [01:00<00:16, 10.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 343/500 [01:00<00:18,  8.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3157894736842105, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 345/500 [01:01<00:16,  9.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 347/500 [01:01<00:22,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 348/500 [01:02<00:35,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 350/500 [01:02<00:30,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 352/500 [01:02<00:29,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 355/500 [01:03<00:21,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 358/500 [01:03<00:17,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 360/500 [01:03<00:20,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 362/500 [01:04<00:20,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 363/500 [01:04<00:21,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6363636363636364, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 365/500 [01:04<00:28,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.03333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 367/500 [01:05<00:19,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 370/500 [01:05<00:24,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 372/500 [01:06<00:28,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 375/500 [01:06<00:21,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 379/500 [01:07<00:14,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.0625, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 381/500 [01:07<00:12,  9.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 383/500 [01:07<00:12,  9.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 385/500 [01:07<00:13,  8.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 386/500 [01:08<00:19,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 389/500 [01:09<00:25,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 390/500 [01:09<00:23,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.11764705882352941, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 392/500 [01:09<00:19,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5217391304347826, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 397/500 [01:10<00:12,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 401/500 [01:10<00:09, 10.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 403/500 [01:10<00:11,  8.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 405/500 [01:11<00:13,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 407/500 [01:11<00:15,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 409/500 [01:11<00:15,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 411/500 [01:12<00:13,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3636363636363636, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 413/500 [01:12<00:15,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 415/500 [01:12<00:10,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 417/500 [01:13<00:11,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 420/500 [01:13<00:11,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 424/500 [01:14<00:10,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 425/500 [01:14<00:17,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 426/500 [01:14<00:17,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▋ | 432/500 [01:15<00:08,  8.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 435/500 [01:15<00:06, 10.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 437/500 [01:16<00:09,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 439/500 [01:16<00:08,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 441/500 [01:17<00:10,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 444/500 [01:17<00:09,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8235294117647058, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 447/500 [01:17<00:06,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 449/500 [01:18<00:07,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.7499999999999999, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 451/500 [01:18<00:06,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 452/500 [01:18<00:08,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 456/500 [01:19<00:06,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 458/500 [01:19<00:05,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 460/500 [01:20<00:07,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 463/500 [01:20<00:04,  8.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 465/500 [01:20<00:04,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 467/500 [01:21<00:05,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.08333333333333333, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 468/500 [01:21<00:05,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 469/500 [01:21<00:06,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.47619047619047616, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 471/500 [01:21<00:05,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 475/500 [01:22<00:02,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 477/500 [01:22<00:02,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 479/500 [01:22<00:02,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3846153846153846, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 483/500 [01:23<00:03,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 488/500 [01:24<00:02,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.046511627906976744, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.375, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 491/500 [01:25<00:01,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 494/500 [01:25<00:00,  9.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 496/500 [01:25<00:00,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.1, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 497/500 [01:27<00:01,  2.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 499/500 [01:28<00:00,  2.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 500/500 [01:40<00:00,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344294b110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e248d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2040>, 9883885.931112433)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344294ad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3230>, 9883885.970452735)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442fa0050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534428d61d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8007c0>, 9883889.416357808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428d64d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429640d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442873250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442805550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534428b9190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8022e0>, 9883888.596422192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428b9850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dcad0>, 9883888.63398444)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428737d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cb540>, 9883888.13675181)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442964210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cadd0>, 9883888.202582253)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442853190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344283a050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4def20>, 9883888.339665469)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428a4750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427aa910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2c10>, 9883890.924600303)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443009510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344277b3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a930520>, 9883890.595105015)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344277b4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427f7d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a932270>, 9883891.96354941)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344266b790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427ce6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427f4490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9312b0>, 9883891.024695864)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427cee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a803230>, 9883890.77079462)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427f4bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344283b1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9da120>, 9883897.090303743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426d5110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442cd1810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536430>, 9883895.408383045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534425567d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442443c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed6350>, 9883895.874496719)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442443b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344240b5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a932f20>, 9883897.746072764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344240aa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442488350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed5b00>, 9883896.117162935)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442488210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425b1190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed6660>, 9883896.688541628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442496e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344235cc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed69e0>, 9883898.509122252)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344235cd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425f7a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c5630>, 9883899.602686567)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344224ced0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442227d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c7cb0>, 9883901.889397925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442233a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426f0fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534423e7810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442203110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442225650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442234d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed56a0>, 9883898.921166124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442414550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c5b70>, 9883899.870721668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442225510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c6d60>, 9883899.634931251)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442202f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c4d70>, 9883898.276052348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534423e76d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c63c0>, 9883898.968617285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442327650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344212e890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c06e0>, 9883912.8638428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344212eb50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442389990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed7000>, 9883901.440958492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442402b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534422e8050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a931550>, 9883901.915238945)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534422e8a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421f4310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a1860>, 9883904.329962905)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534421f4650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344215a2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442180110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442197d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421b3a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c38c0>, 9883902.012672607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442180750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a0910>, 9883901.9282421)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534421b3f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c1010>, 9883902.4499623)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344215a0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a0360>, 9883902.123560973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442197c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421de990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208d320>, 9883905.861468168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534421de350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534420db8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208d160>, 9883905.578257432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442153810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f35b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208dcc0>, 9883905.071020996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f35d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f888d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8c210>, 9883905.952274565)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f88790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421c4d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344202ab50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344207fa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344200e590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534420ec850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441eff690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a02f0>, 9883904.488818176)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344207f8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a35b0>, 9883904.510759892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534420ec650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8cbb0>, 9883910.980515113)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f90b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a3bd0>, 9883903.69387728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344202abd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208c4b0>, 9883904.123836452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344200ea90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208e120>, 9883904.931088174)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441eff550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441e16690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8eb30>, 9883908.604475709)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e16910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441e76450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6de10>, 9883917.589502648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e767d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441e8d990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6cad0>, 9883909.283098249)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f9a990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f6ed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441fe4790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441fe70d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a1a90>, 9883906.909784455)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441fe6f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208c130>, 9883907.259204764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f6e010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8dbe0>, 9883906.84543656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441fdee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441e66290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6f000>, 9883909.543251475)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e66990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534420ab390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d70de0>, 9883911.279755253)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534420aa850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ddee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d722e0>, 9883911.567754276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ddf190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d4a950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d5f9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d700c0>, 9883909.87121948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d5f890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6f850>, 9883909.692262916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d4a750>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ebf790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5c980>, 9883912.207253795)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ebfe10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c40610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d731c0>, 9883912.212072683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c403d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534420b8c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5f7e0>, 9883914.600844437)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b1bfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c92050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5cf30>, 9883912.122924348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c92710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c832d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b54830>, 9883915.024880659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d22510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b0db50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b56e40>, 9883916.35554745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c79fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ad9ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a46970>, 9883919.180772856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ada250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441aeec90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a470e0>, 9883920.27687554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441aeeb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a2cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a2e290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6e890>, 9883915.563788336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a2ca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5f700>, 9883916.01669242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a2e250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344194b8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441938910>, 9883918.422790164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441af4e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419818d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419a1fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441939d30>, 9883920.404015029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ade350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193a0b0>, 9883919.148034379)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534419a1e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419d6550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419e9690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193aba0>, 9883919.916998062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534419d6c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419e9390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193b2a0>, 9883920.46236618)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534419eb810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441930ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a475b0>, 9883919.870492524)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441930c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441803010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534417ffc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d726d0>, 9883919.8876612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441802fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441828b40>, 9883921.536135988)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441800c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441914b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441828d00>, 9883919.461309116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344181fa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344204c450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441885e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441bbb790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a4a990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193b0e0>, 9883919.479152624)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534419e9510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a45710>, 9883918.419996843)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a4a910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441829400>, 9883922.263957117)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344204c110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193bcb0>, 9883920.205774125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441885c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441938360>, 9883917.634719877)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441bbb750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441891690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442de5c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442990290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442968650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429d5810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442cbfdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c34550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344281da10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344281f450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344283a1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442815ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429aa790>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426fd690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534428a7b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442715d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344272f890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344281f990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442890dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442790690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427a2f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442888190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442cbc210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442642ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442610450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426683d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344266e790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426b5910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426cd890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427f5250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344262e910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426dcb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442501f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442528910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344252bf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425543d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442569450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442584b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425bf4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425eb310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534424024d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442408810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344242e1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442415490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442441a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442495210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442529790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534424e30d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442305c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442313210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442334550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427a3110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442354750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442389a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344237de50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534423a2b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534423c8e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442262d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534424c0590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534422b6690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534422cf350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534422f34d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442279d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442105e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344212c950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442133510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421dc150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c12110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534420405d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344201f390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421c5090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534420a6950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442058b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442000fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442041890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f72ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f99890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441e14690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441fc7190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441dfce50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ea2d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ebfdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534420aba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441e253d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441e8bdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d115d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d18a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d3a510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d88250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d12190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c06190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d9c390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d4bc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c52610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d750d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c353d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f4a350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441cc4090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441cece90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c7a050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b19b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c93210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f19110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b872d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b95010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441baf090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441cd1410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b7b990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441bbabd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a09010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a32790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a584d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a88c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a48fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441aa0a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ab9d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ac2410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441914450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441926f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419cddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d72970>, 9883918.4250287)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441903a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344182a3c0>, 9883921.287356853)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441891150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3000>, 9883889.157252364)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442de6090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f22e0>, 9883886.783054136)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442990610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b4e50>, 9883886.535741301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442968a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4deb30>, 9883886.860900346)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d30e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df2a0>, 9883889.60707034)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c96d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df460>, 9883889.604118744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c348d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dd940>, 9883887.791853052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344281e0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dee40>, 9883888.016023956)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344281f650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dfd20>, 9883889.133072944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442839fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b5f60>, 9883889.404511308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442815d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e18d0>, 9883892.046643632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442891ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a801080>, 9883893.187756348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426fd550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b60b0>, 9883892.483204233)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428a7c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a800a60>, 9883889.520995775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442715c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a802d60>, 9883890.109081808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344272f550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e0670>, 9883890.312135687)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344281f810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a803a80>, 9883890.409140574)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c4df50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8001a0>, 9883890.579693602)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e08cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f10f0>, 9883893.092583332)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427a2e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a932430>, 9883891.478955915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442888290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e2ac0>, 9883891.67514746)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442cbf210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a932a50>, 9883893.625840936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442642990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a931470>, 9883906.72016828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426102d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9339a0>, 9883892.428767228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426688d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9331c0>, 9883894.059425509)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344266e610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a933690>, 9883892.825772917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426b5550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8008a0>, 9883893.00371239)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442888410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadeee40>, 9883893.400223482)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427f5390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a932d60>, 9883893.360136716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344262e710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadecfa0>, 9883895.698072976)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426dd150>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadef700>, 9883893.071299069)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442502590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ddb00>, 9883895.391948616)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442528dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadecd70>, 9883894.723449802)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344252be90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9d8520>, 9883894.419568907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442554910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df620>, 9883894.1501343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442569b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f39a0>, 9883895.305820595)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344257ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb2b30>, 9883895.62293063)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426b6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458537310>, 9883894.494893307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534425eb110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadec440>, 9883896.41563792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534424022d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a800de0>, 9883896.201503461)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344240b310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b49f0>, 9883896.279695123)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344242e910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed4830>, 9883895.022111287)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442415790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a802eb0>, 9883896.242647031)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442441d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533faded400>, 9883896.978526803)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442495010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a930670>, 9883899.642717581)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442541e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c4910>, 9883897.140264949)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534424e2d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c5240>, 9883897.832047824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534423058d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c55c0>, 9883897.740126751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442313010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c5e10>, 9883899.831242047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442334b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed6200>, 9883897.961404445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427a3910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c67b0>, 9883908.053112527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442354e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c74d0>, 9883899.463008257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442389810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533faded860>, 9883898.370889343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344237dd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c7cb0>, 9883900.604330797)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534423a2a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c4520>, 9883900.600306744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534423c8c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadee120>, 9883899.885197448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442262bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c7930>, 9883901.810899835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344231d210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c0360>, 9883900.280276733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534422b6f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c0d00>, 9883901.291507104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344241f990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c14e0>, 9883911.856177345)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534422f11d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c7700>, 9883901.348024981)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442279950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c18d0>, 9883901.777620515)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442226350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c23c0>, 9883903.172431726)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344212cdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a0750>, 9883902.550783675)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442203e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a0f30>, 9883903.995466435)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534421dc810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a1630>, 9883902.649759362)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442324950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c34d0>, 9883904.029241616)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442040cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a3150>, 9883906.345685195)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344201ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f1630>, 9883904.165403288)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442227b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208c6e0>, 9883904.900923235)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534420a5210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208c980>, 9883905.066157442)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344200f690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208de80>, 9883905.162749056)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534420eee50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208fb60>, 9883906.620680591)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f417d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a26d0>, 9883906.202526487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344224ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8cd70>, 9883906.229396252)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f99750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a1a20>, 9883908.74590274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e14490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208c1a0>, 9883908.941679467)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441fc4550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8f4d0>, 9883907.409972636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441dfca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6cf30>, 9883909.331256228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ea2c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208c280>, 9883910.398998236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ebee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8f8c0>, 9883907.921830295)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534420ab4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8e890>, 9883908.90527626)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e24650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8d010>, 9883907.971317256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e8bc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a2740>, 9883909.217073934)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d11b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6eac0>, 9883909.794838)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d19110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadee190>, 9883913.84762242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d3b190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d707c0>, 9883910.618893648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f18fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8ef90>, 9883911.952403162)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d8a650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8edd0>, 9883911.32933778)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e744d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d72f20>, 9883914.056197846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d9d350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d71a90>, 9883912.202234907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d4b350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5c050>, 9883912.35788785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c52b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d70520>, 9883910.855996264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d74690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d71550>, 9883912.945136145)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e66e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208f620>, 9883914.88142088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e74190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5db00>, 9883914.949148096)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d4a9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5e820>, 9883913.231858833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ced650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5da90>, 9883913.2002001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c79f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5f230>, 9883914.231606876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b19910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d72740>, 9883916.788564311)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ddee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c71c0>, 9883913.829056213)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441dde310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b550f0>, 9883916.749420308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b87190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b558d0>, 9883917.903797334)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b95250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6d1d0>, 9883915.326308165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441baee90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5c910>, 9883913.768761093)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441cd1710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5f4d0>, 9883913.529390195)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b7b850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b55cc0>, 9883915.355722671)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b65c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b55390>, 9883917.394777644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a08e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a44050>, 9883916.789899724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a33190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b573f0>, 9883919.207797304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a5bb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a452b0>, 9883916.470943488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a88a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a45400>, 9883917.223519528)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a49190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a45a20>, 9883916.324932076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441bd8fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d72f90>, 9883917.388960436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441aba450>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a46040>, 9883917.398995483)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ac2a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b55f60>, 9883918.421553265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441926d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193a970>, 9883919.13986518)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e662d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426cd2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b41a0>, 9883894.117641062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426cc4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534417191d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441755290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb3b60>, 9883921.87778964)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344171a2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441829010>, 9883922.416413592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441755690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344171bb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d71cc0>, 9883927.392194195)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441719350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419cd950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441802790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a3540>, 9883923.82861505)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441803d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ebdcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a46d60>, 9883922.573923944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ebc490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441fd6190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534420a74d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c5d30>, 9883922.435912896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441fd6050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c4e50>, 9883922.828865612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534420a6c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442292e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a932970>, 9883923.844094254)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a391f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344177ed50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b5b910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441829a20>, 9883922.761600537)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441afcf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a802580>, 9883924.364433696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534424c9c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534417fc490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425e98d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0210>, 9883923.981856938)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534417fda50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b3620>, 9883924.274695957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534425e9210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427e9d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344289f190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534418f0850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534418f39d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534417038d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534418608d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344172db50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a47690>, 9883921.850909386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534417310d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193a740>, 9883921.464950109)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441703790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208dbe0>, 9883924.564973194)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427e9ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b35b0>, 9883924.11547074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442510490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458436190>, 9883920.596153112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534418dfed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f19470>, 9883922.164917754)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534418f37d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a46900>, 9883921.278595047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441adafd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6e9e0>, 9883921.258425154)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344172dd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429830d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442992590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0926d0>, 9883924.870137371)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442980f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534419394e0>, 9883924.92028476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442992410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c49110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0917f0>, 9883927.581059432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442678b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344177ebd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c6350>, 9883924.507570973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344177db90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c86a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442cc8190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a091390>, 9883926.127494777)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344177c750>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9da0b0>, 9883925.854959035)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344266f490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c0d790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c35150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b3f50>, 9883926.058201663)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ebd590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a093000>, 9883925.754521215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c35810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344177ffd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a092190>, 9883925.69046081)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c7e550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442df0e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030b000>, 9883926.17587415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441730a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d47c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d6cd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442769990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344291e110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442911f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b2f90>, 9883924.051255656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344291cc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0de0>, 9883924.179935368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442912310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c3150>, 9883927.617242329)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d47690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae18c20>, 9883927.335352996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d6f310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b3850>, 9883923.785104744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428b8910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419eb990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae18360>, 9883927.20851515)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e73810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c035d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19470>, 9883926.513745012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e82090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442fed510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e09190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a47af0>, 9883928.698175082)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442fef110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19240>, 9883927.790184774)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e08d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c02a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f4f790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae183d0>, 9883927.081653493)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c002d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5e580>, 9883928.683150783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c1e890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534430cd3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d390>, 9883930.054780422)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430cee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ea0a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443029250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f770>, 9883928.95637284)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459f20750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344309d010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f460>, 9883929.984544188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344172e090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443060310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443076bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f1c0>, 9883929.3093916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443063bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f3f0>, 9883929.568896111)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430773d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d540d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e24850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f45f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030b310>, 9883927.734856987)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443028c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae1b230>, 9883927.404821636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f46b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421f7950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a090830>, 9883927.63802024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d54290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5ef20>, 9883927.14972622)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e26a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae181a0>, 9883927.537316717)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534420e1d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d6c750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae1a580>, 9883930.26564342)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534431a4fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534430540d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3af0>, 9883933.7057908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443057210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431ea6d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344300b4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a46e40>, 9883930.710248793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534431e9650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1860>, 9883937.12769124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344300bed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443046150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344315bd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3930>, 9883930.115088608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ebdb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1a20>, 9883932.942928256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443158810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442ed9b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc8b1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0130>, 9883931.625931447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ed9650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0c90>, 9883930.5828959)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d544d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc6a4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441829c50>, 9883931.840931242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc68290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e09b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1010>, 9883930.86907305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e09790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429d6ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc15390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0919b0>, 9883931.277731547)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429d4e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbec150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f25f0>, 9883932.307379665)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbee990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc4a310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534417021d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e9d290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345983c520>, 9883932.27028988)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc16350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f1b150>, 9883932.217457816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc1f410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431b9410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a092dd0>, 9883929.914272947)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a16e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f26d0>, 9883928.939166635)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e9df90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8db230>, 9883929.619934924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534431ba010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3f8650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f1550>, 9883932.611427529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3f8890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f700>, 9883939.21330658)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534421b1590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5e190>, 9883939.506614763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344236af50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e7afd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e30190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc56510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e39c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d407710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3edcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a093a10>, 9883933.94006732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e79410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f0bb0>, 9883932.94653192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc89bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0830>, 9883932.03035553)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344307c590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f31c0>, 9883931.818830244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344282a710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f0910>, 9883932.806806272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d406e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f3cb0>, 9883932.935171587)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3ef010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427ab710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534428ea910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534428d4dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427a3a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442912910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c12190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d3c950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344308dc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5cad0>, 9883939.726086313)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344308cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f35210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc54750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344310ab50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d360850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc08310>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3eefd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3f8690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbc4090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3adc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3a10>, 9883940.787249472)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534431f5790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0fa0>, 9883939.338437883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ddf4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0ec0>, 9883940.257425599)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3ad690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f070>, 9883940.590057304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427c0e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d470>, 9883941.582852768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428eb4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d7f0>, 9883940.463225933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427e1cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5cfa0>, 9883939.540674377)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428adfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5eb30>, 9883943.772755316)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442911b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5e660>, 9883939.15746277)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c13110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5da20>, 9883942.630385676)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d487d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3000>, 9883941.296446761)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f52c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2d60>, 9883940.015457656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc6c090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0440>, 9883939.577560766)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443109590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1a90>, 9883940.17582862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3626d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2040>, 9883940.240964368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc08c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f04b0>, 9883939.35561997)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3ec990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d546690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3460>, 9883939.754609656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d544650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d577990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d532d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9dcc0>, 9883941.730059212)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d533850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d70f290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2dd0>, 9883945.005769625)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d70f790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d66b490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9f2a0>, 9883941.472655972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d66b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d537c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d56f950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a3070>, 9883943.64202027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f1e690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9fe00>, 9883941.549353454)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d56c250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d52b150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c4b0>, 9883943.155488074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d672b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d71bf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a833410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c750>, 9883942.184746588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a813910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9cec0>, 9883942.611412045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a8329d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d553310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c360>, 9883942.25038132)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d553250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345f66e2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc95710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9e890>, 9883942.802108904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d69ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b5aac0>, 9883942.671017846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc96e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d5ac150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431426d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b5a270>, 9883942.688098269)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbe4ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b581a0>, 9883944.294072067)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443142750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3b4050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208ce50>, 9883941.98420011)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d574750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b5bee0>, 9883946.33515086)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d54b190>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbf7b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5c280>, 9883943.290820396)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbf7dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d56e850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b3b60>, 9883944.36643868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d561210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442888750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fafcd650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b58980>, 9883946.407315912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442864690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d70b090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443119f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459302200>, 9883947.138441848)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d708b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df770>, 9883944.676777748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d5a7c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a853110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9e660>, 9883944.919204412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a853190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345999d3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a7d8dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6cfa0>, 9883949.352779685)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a52f690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f14e0>, 9883946.392502952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a7dbf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345ad75ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f3620>, 9883945.41654038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345ad765d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a544290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345baee310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2f20>, 9883946.55214004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a5478d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f3a10>, 9883946.956897832)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345baee410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a4f40d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fad7c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f0910>, 9883945.821123788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a6b0910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a2c11d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d4d5b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f1710>, 9883943.978332479)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fafcf050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b6f90>, 9883945.691513067)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fad7d610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f25f0>, 9883946.039565623)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a2c3590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b52b0>, 9883946.192655258)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d4d6e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a68a650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b6f20>, 9883951.703863604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a688c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a796390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7620>, 9883946.536760552)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fadd9c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d51b510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3af0>, 9883947.034550205)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d51b150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a029850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a0cbf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e0980>, 9883948.072961483)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a02b250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b4d70>, 9883952.188687684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a0c80d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a2b9950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b4590>, 9883949.92401254)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a28fe10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345f586890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e1f60>, 9883949.181071328)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3245d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15358dcee910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c87c0>, 9883949.858458018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d4cf610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a044850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e2510>, 9883948.650581604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a045f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459b74950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fadbfb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a2f2150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d4b2ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a577790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459302dd0>, 9883946.371468425)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a577e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ca970>, 9883950.2423014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459b748d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8050>, 9883949.34075032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fadbe710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b6c80>, 9883945.991568932)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a2f0fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7e00>, 9883946.96429768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d4b2e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d4b40d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c9550>, 9883951.715397857)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d4b5310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534599b8490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d503e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c9cc0>, 9883950.175997697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534599bb710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459c4b550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e755c0>, 9883956.284007777)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534596dc990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534594f7350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e75b00>, 9883950.559878226)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459370210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345f650e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193a200>, 9883949.171682112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345f652250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443e37950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e77620>, 9883950.026609372)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443e36c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443fcb6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e77930>, 9883952.913618311)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443fcb190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f5ec10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a397f0>, 9883955.919591196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f5e390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d483f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cb150>, 9883951.087187476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d483d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fad46990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534581c9f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459f42810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d51a9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e75710>, 9883951.252115125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a083450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d480850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429a42d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e76c10>, 9883949.9288995)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345ad4cb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19bc40>, 9883951.378561104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a460fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19a190>, 9883953.246678198)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581cae50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208c210>, 9883948.575514784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d33cc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9f3f0>, 9883948.465374785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d518f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e77700>, 9883949.246445196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d4834d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458184d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e76900>, 9883953.682426602)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581871d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458295050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19ac80>, 9883952.622911526)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458294d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458249f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a199780>, 9883953.048702858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345824bad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458248150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583c3a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19b700>, 9883954.036645617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458248490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cb380>, 9883953.67373478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534583c3e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345838f210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459301a90>, 9883958.881206572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345838ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345834d050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458354ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5c280>, 9883953.964033436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458356f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534582fd990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5f460>, 9883954.876491388)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534582fd590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fad37f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a1990f0>, 9883955.906981751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534582aaa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d50ddd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f8e850>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345829de10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345826d590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9f540>, 9883953.88921646)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345adccc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534580e9050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e74830>, 9883951.912095847)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d50c910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e75470>, 9883951.984075136)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534580eb7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c40c0>, 9883952.11722662)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345826f590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5e200>, 9883955.90816192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d46d550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a1996a0>, 9883952.34131142)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f8c690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19a3c0>, 9883951.829181155)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459f601d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a450250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458430e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5def0>, 9883954.65985559)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d332450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583e6e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cba10>, 9883955.838705381)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534583e5590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585bd250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a390f0>, 9883960.807299657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585bf790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458593990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458532650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a39630>, 9883956.44816594)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458593e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3b150>, 9883956.089721693)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458531810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458514c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3b700>, 9883956.206677085)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458515910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534584eca50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5ef20>, 9883956.900625024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584ee150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583b7910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534586584d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a383d0>, 9883956.835890535)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585b10d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5d240>, 9883957.878971403)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458659f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345834ea10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458633b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a5f0>, 9883956.953261856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345834e190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a39c50>, 9883957.232445039)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345865a1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345849a090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993ee40>, 9883957.144291632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345879b490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534584c6150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a199010>, 9883955.9544638)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a907950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345824acd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19a7b0>, 9883954.89030812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584c6950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e76ba0>, 9883956.682391424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584828d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a38e50>, 9883954.940237613)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345824b250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345857cad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5f850>, 9883957.974982653)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345857f110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587bb7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993e580>, 9883957.710046548)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587bbb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458741cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993c130>, 9883958.456029644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458743190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458702f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345866b590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993c980>, 9883958.939239373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587013d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993e7b0>, 9883958.020840136)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585159d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534588ba1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345887a0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e75a90>, 9883958.962917585)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534588b9950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993d320>, 9883959.204353089)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458878790>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587e8750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534595130e0>, 9883959.2646127)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587ea050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458800b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513070>, 9883958.254897932)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458800750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587e72d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534589a4e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459512970>, 9883960.018512549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584307d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993d0f0>, 9883959.403709153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589a5490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458951290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534595115c0>, 9883959.175360484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458963190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458978350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458927e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a350>, 9883959.923973465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458978110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534595110f0>, 9883959.010799723)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458924c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345867d210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534595112b0>, 9883959.787194544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345867c690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458903390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a96d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459511e80>, 9883965.682379818)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585a1a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a45150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a02e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458879a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993e900>, 9883962.281557793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458742c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b855d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973b000>, 9883967.321964089)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b93190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b50cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513770>, 9883962.944008924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b518d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587039d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b0d390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459739a20>, 9883961.637930874)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b2ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a430>, 9883961.929099204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b0ddd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ba0c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a190>, 9883961.986300232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ba2590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343621b690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c7d450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a900>, 9883961.882023724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343621bc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c3ddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993c3d0>, 9883960.305144656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a97090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993dd30>, 9883960.826974163)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a45490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3b7e0>, 9883962.138739975)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458803390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534589fc110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a6c950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5ce50>, 9883959.730150767)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589cee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a198130>, 9883962.66552722)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f5ec50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a9e0>, 9883961.226017075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c3cbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a7b0>, 9883959.73591866)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589fcd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458533710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c3d0>, 9883963.066160023)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534586f8090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a9e450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c750>, 9883962.06872782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458615110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345890bc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d92d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7070>, 9883962.765978888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589091d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962dcc0>, 9883964.97228872)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d90690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d51290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962cad0>, 9883962.94880102)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d52910>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c3e750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d23d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c050>, 9883962.996922052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c3fa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459512890>, 9883964.121612856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d21950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b0a850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459511ef0>, 9883963.346184475)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ed2910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e5add0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458dfa650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962e970>, 9883964.956795152)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e5b310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8ec80>, 9883964.156334529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534596a0950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345841c190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e26650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8d390>, 9883965.409815587)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e90650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e9e0>, 9883969.515008297)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e27890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458cfd090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458cd9dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fad26f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458be4190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962e820>, 9883962.159849908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458be5850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962f7e0>, 9883965.257777095)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e132d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459739b70>, 9883965.454886813)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458cd9d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962fee0>, 9883961.992263045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fad25f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d22150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f77210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962cd00>, 9883966.783879302)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f77e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f07a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8fd90>, 9883968.236787843)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f07b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534596c60d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8c7c0>, 9883966.989534102)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d48310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e4c390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459090b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8c520>, 9883966.449561767)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e4ea50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345907ad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d755c0>, 9883966.204613188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345907a890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345901d610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973af90>, 9883966.227514286)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459016e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f3d8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d74830>, 9883968.227766164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534588a8a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e91490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345841f2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d74280>, 9883968.31852427)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e92750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c830>, 9883967.124937965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345841f110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459316410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a38520>, 9883966.5262404)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d20c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e02e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458efd490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8cde0>, 9883965.929480076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459090f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a00450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993d390>, 9883965.352436194)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f28950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae2270>, 9883967.361534417)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459317250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae0130>, 9883965.245767027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e03790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8c910>, 9883965.483367492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534590c3450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459532150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d76f90>, 9883969.706774509)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d1a590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344174a6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863c050>, 9883970.658920964)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534417486d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344176e090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863fee0>, 9883969.544043606)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344176f690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441177c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344118fdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863cec0>, 9883969.234613348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441177ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75f60>, 9883969.562739024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344118fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534411a3250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863c600>, 9883972.25698883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534411a3990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458efd210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863eb30>, 9883969.373698913)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458efcf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534411b6a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863de80>, 9883969.596452508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534411b66d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459127010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534411e6690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738ad0>, 9883969.625677828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459063b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435320>, 9883970.55831755)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534411d9f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410011d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae29e0>, 9883970.396824004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441000fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441002fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459170590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345925d0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458fe9190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441175690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863c210>, 9883967.545591015)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345911d6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863ee40>, 9883968.311756631)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441175550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d748a0>, 9883970.415365512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441002d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d77460>, 9883968.331080856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c11c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863cbb0>, 9883968.527350513)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534590a7450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441026f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458434210>, 9883978.999510922)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441026dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e89c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441065910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435860>, 9883970.63484718)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f07850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458434830>, 9883975.6147797)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441066010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344100ba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410985d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747d20>, 9883972.041762142)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441067350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738590>, 9883970.9431848)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344108ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410c6910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c590>, 9883976.657539807)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534410c6fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410877d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410d98d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863fd90>, 9883977.160808496)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459752b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745860>, 9883972.40095236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534410d96d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344109dd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410e7710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747cb0>, 9883971.597762741)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344109e3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745390>, 9883971.88518926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534410e75d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f06950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747a80>, 9883971.82393464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f06790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f15710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458746e40>, 9883972.427755332)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f15e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344109ec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441043dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441055c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8c280>, 9883972.599706644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344109ecd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584348a0>, 9883970.757152263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441043c90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435c50>, 9883970.439346172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441054750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f4fa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534587442f0>, 9883973.180716025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f4f950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f676d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a737e0>, 9883973.165609691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f64f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f30490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a73000>, 9883974.25756928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f305d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f97190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440faa5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a71e80>, 9883974.138853475)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345893f990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440fb4110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863dfd0>, 9883973.839963716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440fb4610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410f0490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962def0>, 9883978.097612187)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f28c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344101ca50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e12110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458861010>, 9883974.015260737)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344101cad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534588625f0>, 9883976.128556252)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e11fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e21890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458861da0>, 9883975.158308333)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e21ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e3ca50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458860830>, 9883975.78331012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e32810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f2c750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e56bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458861630>, 9883975.788027927)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440fb4ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458861e80>, 9883976.691473372)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e56a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e08f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e76550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f8f5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435470>, 9883973.496360255)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440faaad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a70910>, 9883972.854389528)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f8cc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458862040>, 9883974.967634434)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e09590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458436e40>, 9883975.926636497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e76e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534411755d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a71550>, 9883972.459888373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441175ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e96450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a70bb0>, 9883976.423634307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e966d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440feba10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458863a10>, 9883976.775895506)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ec1350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ed6590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ee5f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a73ee0>, 9883978.098887447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ed4750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458746200>, 9883976.568224523)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ee5dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ec0390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896d400>, 9883976.165749691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ec0890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d2fd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896e6d0>, 9883977.134299949)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d2fbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d43850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8ec10>, 9883980.89052832)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d43710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d80610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7b070>, 9883977.85109898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d80b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e88e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437d90>, 9883979.4017963)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d65d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e12f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440db9c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ead890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440fab610>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d11210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458863620>, 9883976.825423451)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440fab550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896d0f0>, 9883976.28535946)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d110d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7bd90>, 9883978.178818656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e13250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7aba0>, 9883979.240540357)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440db9a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458861cc0>, 9883976.488595096)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ead590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440dc5390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458863d20>, 9883979.982303556)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344109ad50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f95710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b78210>, 9883978.521436457)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440dc6c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bfded0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c10e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b79710>, 9883980.049201611)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e3ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c3a190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896f150>, 9883979.91870218)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c3a050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c4cf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c563d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8f700>, 9883980.778901633)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c55f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440de8050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c80e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896eb30>, 9883984.555746872)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440de8650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7b690>, 9883980.869619155)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c80c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344100bd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745e10>, 9883980.485720528)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344100b5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410dacd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440cbd090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896ea50>, 9883980.658516267)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534410d99d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458537770>, 9883981.900097491)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440cbc390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440cd0e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896cfa0>, 9883979.225922544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bfe590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440de1290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d38cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a350>, 9883982.22015246)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c4b910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c42590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896f310>, 9883978.841942592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d12550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458535780>, 9883981.442499762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440cd0c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a5f0>, 9883978.88748422)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d390d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7b540>, 9883978.963223767)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441005f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b6cd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458317e70>, 9883982.44316951)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b6cb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b09f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a70670>, 9883982.354905432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b082d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d81b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b9cd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458316b30>, 9883985.947138349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d81710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9fa10>, 9883982.455210144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b9cb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bb3f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585349f0>, 9883982.686364535)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bb3e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bd8790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9fd90>, 9883985.313903917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bd8a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b11350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437bd0>, 9883982.63799318)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b11050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b582d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b08710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b1ce10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b36710>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440cea450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b58290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437620>, 9883980.995780423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441026cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440cde1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458315400>, 9883981.396873988)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b1d490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536f90>, 9883981.224054808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440cdddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458316510>, 9883981.226021668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ce9ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314d70>, 9883981.511804763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b08bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9f0e0>, 9883984.215372248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b58410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314de0>, 9883981.479963653)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b58650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c575d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9da90>, 9883983.504804833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c4cfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bbfd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9d320>, 9883988.659209255)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bbdf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bef910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9e970>, 9883985.095913742)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bef6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a7c790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a83350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458534c20>, 9883984.476099696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a7c710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb3690>, 9883984.363349888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a80690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c4b710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440aa5ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1080>, 9883984.539573343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a91b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb22e0>, 9883987.480530912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440aa5cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ad10d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ae6e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ae8710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb0d70>, 9883985.237999348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bbd250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ac4790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344090ad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1860>, 9883984.460432213)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ac4810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb2cf0>, 9883986.260516789)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440908550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ab4ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344092e0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314830>, 9883984.767663112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ab6150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb2c80>, 9883985.463819953)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344092df90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b3ae50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344094f590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a41c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a5bc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9c9f0>, 9883985.9047046)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ad0f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536900>, 9883984.64300757)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ae6cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9e430>, 9883994.699209241)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b39c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9dd30>, 9883987.030333249)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b3acd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3230>, 9883986.983568173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344094fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9e740>, 9883983.455185706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a422d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344099ced0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc32a0>, 9883995.780419484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ac44d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534409b1f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb10f0>, 9883986.471136538)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534409b2250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440970c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534409e4f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb0980>, 9883995.007353228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a91010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3690>, 9883988.913247803)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534409e5490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344098ca10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584347c0>, 9883986.91961487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344098c590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bbd450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9cfa0>, 9883987.46421478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bbd510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440839c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590db7e0>, 9883994.873350741)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440839a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440806690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590dadd0>, 9883995.393083116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408066d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408508d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590dbd20>, 9883987.611540344)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440850790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407fdb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440884650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1240>, 9883988.127637353)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407fe450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d81a0>, 9883995.203243444)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440884d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408b0c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344098ed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534409c2110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a00490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0ec0>, 9883986.499120396)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534409c27d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0360>, 9883988.499382768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408b10d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc1e80>, 9883994.920929374)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344098ec50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9f620>, 9883986.708094906)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440925150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534409eab50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941cc20>, 9883990.668178476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534409eaf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344099f950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440761210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941f620>, 9883990.641913967)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440826e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941d400>, 9883991.960353935)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407610d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440775910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344079afd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9fbd0>, 9884002.00978315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440775690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d88a0>, 9883992.85123622)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344079ae90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407b2290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c9e80>, 9883991.480436368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407b2910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407c2e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c8de0>, 9883991.44605056)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407c3190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440763350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408ec550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408ee790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440710290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440715090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440748310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408cde10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3a10>, 9883991.88524732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408cdc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc1d30>, 9883989.538570132)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440710750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941c980>, 9883992.354877181)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bd0210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941f7e0>, 9883988.957193837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408ecc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc15c0>, 9883989.88903417)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408ecf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941fc40>, 9883990.106968043)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440edd0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941d390>, 9883989.965549508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440748810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440643ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590ca2e0>, 9883993.704491034)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440643ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407f1990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cb310>, 9883995.176352957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407f1790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440aa6ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440762410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440614e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440733f10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534406332d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c8980>, 9883992.008067492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408ce750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cba10>, 9883993.000457408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407624d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590caba0>, 9883992.955271065)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440615590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d9d30>, 9883992.652725449)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407e3190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c9a20>, 9883992.976086816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440633110>\n",
      "Evaluating workflow:   0%|          | 1/500 [00:01<13:14,  1.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 3/500 [00:01<04:24,  1.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 5/500 [00:02<02:48,  2.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 8/500 [00:02<01:52,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 9/500 [00:03<02:24,  3.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 12/500 [00:03<01:42,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 13/500 [00:03<01:45,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 14/500 [00:04<01:48,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 16/500 [00:04<01:42,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 17/500 [00:04<01:41,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 19/500 [00:05<01:30,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 20/500 [00:05<01:20,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7272727272727273, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 21/500 [00:05<02:17,  3.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 23/500 [00:06<01:45,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 26/500 [00:06<01:25,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 29/500 [00:06<01:01,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 31/500 [00:07<01:59,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 32/500 [00:08<02:03,  3.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 33/500 [00:08<02:23,  3.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 35/500 [00:09<02:26,  3.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.56, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 37/500 [00:09<01:50,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 39/500 [00:09<01:26,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 44/500 [00:10<00:42, 10.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 46/500 [00:10<00:43, 10.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 48/500 [00:10<00:51,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 52/500 [00:11<00:54,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 53/500 [00:11<01:44,  4.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 55/500 [00:12<01:26,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 58/500 [00:12<01:10,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 62/500 [00:12<00:59,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 65/500 [00:13<00:52,  8.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.08695652173913045, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 66/500 [00:13<01:13,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.30769230769230765, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 67/500 [00:13<01:25,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 71/500 [00:14<00:58,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 75/500 [00:14<00:37, 11.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.028985507246376812, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 77/500 [00:15<01:10,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 79/500 [00:15<00:59,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 83/500 [00:16<01:07,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.16666666666666669, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 86/500 [00:16<00:47,  8.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 88/500 [00:16<01:03,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 90/500 [00:17<01:25,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 93/500 [00:18<01:08,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 95/500 [00:18<01:16,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 97/500 [00:18<01:06,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 98/500 [00:19<01:50,  3.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.28571428571428575, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 101/500 [00:20<01:36,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 103/500 [00:20<01:14,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.1904761904761905, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 104/500 [00:20<01:06,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 110/500 [00:20<00:35, 10.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.9090909090909091, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.16666666666666669, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 112/500 [00:21<00:51,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4799999999999999, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 114/500 [00:21<00:53,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 115/500 [00:22<01:22,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 119/500 [00:22<00:55,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.04347826086956522, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 121/500 [00:22<01:00,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 123/500 [00:23<00:54,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 125/500 [00:23<00:44,  8.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 126/500 [00:23<00:45,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 127/500 [00:23<00:54,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7368421052631579, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 129/500 [00:24<01:04,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 130/500 [00:24<01:11,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 132/500 [00:24<01:14,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 138/500 [00:25<00:35, 10.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 141/500 [00:25<00:30, 11.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 143/500 [00:25<00:40,  8.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 145/500 [00:26<00:52,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 146/500 [00:26<00:57,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 147/500 [00:26<01:04,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 151/500 [00:27<00:44,  7.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.7692307692307693, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 152/500 [00:27<00:57,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 157/500 [00:27<00:41,  8.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 159/500 [00:28<00:56,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 162/500 [00:28<00:55,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 163/500 [00:29<00:58,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 164/500 [00:29<01:04,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 167/500 [00:29<00:45,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 170/500 [00:30<00:47,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.7142857142857143, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 172/500 [00:30<00:37,  8.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 174/500 [00:30<00:37,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 175/500 [00:30<00:45,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 178/500 [00:31<01:00,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 179/500 [00:31<01:11,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.09523809523809525, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 183/500 [00:32<00:48,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 187/500 [00:32<00:27, 11.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 189/500 [00:32<00:34,  8.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 192/500 [00:32<00:31,  9.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 194/500 [00:33<00:44,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 196/500 [00:34<00:56,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 200/500 [00:34<00:44,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2222222222222222, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 202/500 [00:34<00:45,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 206/500 [00:35<00:34,  8.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.38095238095238093, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.7777777777777778, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 208/500 [00:35<00:49,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 212/500 [00:36<00:43,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 216/500 [00:37<00:38,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 219/500 [00:37<00:47,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 222/500 [00:38<00:42,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 224/500 [00:38<00:41,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 227/500 [00:38<00:31,  8.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 229/500 [00:39<00:40,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.21428571428571425, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 233/500 [00:39<00:39,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 234/500 [00:39<00:37,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.15384615384615385, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 237/500 [00:40<00:35,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 239/500 [00:40<00:45,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 240/500 [00:41<01:01,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 241/500 [00:41<00:59,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 242/500 [00:41<01:06,  3.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 244/500 [00:42<00:55,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 245/500 [00:42<00:58,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 250/500 [00:42<00:30,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3636363636363636, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.21052631578947367, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 255/500 [00:43<00:24, 10.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.13333333333333336, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.022727272727272728, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 257/500 [00:43<00:30,  7.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 260/500 [00:44<00:47,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 264/500 [00:44<00:29,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 266/500 [00:45<00:34,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 270/500 [00:45<00:34,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 272/500 [00:46<00:50,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5625, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 275/500 [00:47<00:38,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 278/500 [00:47<00:35,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 282/500 [00:48<00:33,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 283/500 [00:48<00:47,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 284/500 [00:48<00:46,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 286/500 [00:49<00:49,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 288/500 [00:49<00:39,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.23529411764705882, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.625, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 292/500 [00:49<00:25,  8.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 294/500 [00:50<00:33,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 296/500 [00:50<00:36,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 299/500 [00:50<00:26,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 301/500 [00:51<00:29,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 304/500 [00:51<00:27,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 305/500 [00:52<00:31,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 307/500 [00:52<00:45,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 312/500 [00:53<00:25,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.47619047619047616, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 314/500 [00:53<00:25,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666665, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 318/500 [00:53<00:20,  8.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 322/500 [00:54<00:22,  8.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 325/500 [00:54<00:25,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 329/500 [00:55<00:26,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 335/500 [00:56<00:17,  9.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3157894736842105, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 337/500 [00:56<00:17,  9.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 340/500 [00:56<00:21,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 342/500 [00:57<00:17,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 344/500 [00:57<00:28,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 347/500 [00:58<00:25,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 348/500 [00:58<00:25,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 353/500 [00:58<00:16,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 355/500 [00:58<00:14,  9.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 357/500 [00:59<00:12, 11.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 359/500 [00:59<00:15,  9.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 361/500 [00:59<00:16,  8.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.045454545454545456, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 363/500 [00:59<00:15,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6363636363636364, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 365/500 [01:00<00:20,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 367/500 [01:01<00:28,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 368/500 [01:01<00:26,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 369/500 [01:01<00:26,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 373/500 [01:01<00:15,  8.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 376/500 [01:02<00:15,  7.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 377/500 [01:02<00:17,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.05714285714285715, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 379/500 [01:02<00:22,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.26666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 380/500 [01:03<00:25,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 383/500 [01:03<00:18,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.631578947368421, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 388/500 [01:04<00:20,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 393/500 [01:04<00:14,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 395/500 [01:05<00:13,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 398/500 [01:05<00:13,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 400/500 [01:05<00:13,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 402/500 [01:06<00:11,  8.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 403/500 [01:06<00:12,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 405/500 [01:06<00:15,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 407/500 [01:06<00:13,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 410/500 [01:07<00:12,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 412/500 [01:07<00:10,  8.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 415/500 [01:07<00:11,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 417/500 [01:08<00:15,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 419/500 [01:09<00:17,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 422/500 [01:09<00:10,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 424/500 [01:09<00:09,  7.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 426/500 [01:10<00:13,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 430/500 [01:10<00:08,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 432/500 [01:10<00:10,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 433/500 [01:10<00:11,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 436/500 [01:11<00:09,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 437/500 [01:11<00:10,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 439/500 [01:12<00:12,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.0909090909090909, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 443/500 [01:12<00:07,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8235294117647058, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 445/500 [01:12<00:05,  9.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 447/500 [01:13<00:10,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 448/500 [01:13<00:09,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 450/500 [01:14<00:10,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 451/500 [01:14<00:09,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 454/500 [01:14<00:06,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 455/500 [01:14<00:07,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 457/500 [01:15<00:08,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 458/500 [01:15<00:08,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 459/500 [01:15<00:09,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 461/500 [01:16<00:09,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 466/500 [01:16<00:04,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 469/500 [01:16<00:03,  8.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.47619047619047616, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 471/500 [01:17<00:04,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5263157894736842, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 473/500 [01:17<00:03,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 476/500 [01:17<00:02,  8.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 478/500 [01:18<00:03,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 480/500 [01:18<00:03,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 485/500 [01:19<00:01,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 487/500 [01:19<00:01,  9.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 490/500 [01:20<00:02,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 493/500 [01:20<00:00,  7.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.375, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 495/500 [01:21<00:00,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 498/500 [01:23<00:00,  2.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.05, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 500/500 [01:30<00:00,  5.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 3/500 [00:01<04:20,  1.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 8/500 [00:02<01:27,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 13/500 [00:02<01:08,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 15/500 [00:03<01:24,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 17/500 [00:03<01:22,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 19/500 [00:03<01:07,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 20/500 [00:04<01:22,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 23/500 [00:04<01:02,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.7272727272727273, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 25/500 [00:04<01:03,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 27/500 [00:05<01:12,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 28/500 [00:05<01:25,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 30/500 [00:05<01:17,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 33/500 [00:06<01:25,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 36/500 [00:06<01:15,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 38/500 [00:07<01:07,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 39/500 [00:07<01:08,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 40/500 [00:07<01:39,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 46/500 [00:08<01:07,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 48/500 [00:08<01:05,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 49/500 [00:09<01:18,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 50/500 [00:09<01:34,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.43750000000000006, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 55/500 [00:10<01:08,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 58/500 [00:10<01:14,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 59/500 [00:10<01:10,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 60/500 [00:11<01:19,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 63/500 [00:11<01:19,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.08695652173913045, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 66/500 [00:12<01:29,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 71/500 [00:12<01:04,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 74/500 [00:13<01:06,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 75/500 [00:13<01:10,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 76/500 [00:13<01:15,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 79/500 [00:14<01:30,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 81/500 [00:14<01:06,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 83/500 [00:15<01:00,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 86/500 [00:15<01:24,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 87/500 [00:16<01:24,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 92/500 [00:16<00:58,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 94/500 [00:16<00:47,  8.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 96/500 [00:17<00:59,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3157894736842105, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 98/500 [00:17<01:08,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 100/500 [00:18<01:15,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.1904761904761905, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 103/500 [00:18<00:44,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 105/500 [00:19<01:40,  3.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.9090909090909091, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 109/500 [00:19<01:12,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.27272727272727276, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 110/500 [00:20<01:12,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 113/500 [00:20<01:00,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4799999999999999, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534406b9c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d9be0>, 9883996.300823541)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534406b9e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440558a50>\n",
      "Evaluating workflow:  23%|██▎       | 114/500 [00:21<01:32,  4.19it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440574cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff2900>, 9883996.029327312)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440575390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff3c40>, 9883995.732334644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440559110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440427d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201940>, 9883998.162508752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440479590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440452750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582038c0>, 9883998.188517235)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440453ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440638750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404c9e10>, 9883999.543715104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344031be90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534404bf7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404ca890>, 9884000.145295188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440354950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344048bad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534404af250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458203c40>, 9883998.32624106)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534404af110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582011d0>, 9883998.325056704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344048b990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534406d6ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb0bb0>, 9884001.375082497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344039b290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440894a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c0fa0>, 9884003.42503614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440529c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534402bd9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a4de0>, 9884003.001349645)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534402bd8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440273a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404cbbd0>, 9884006.552377287)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440272f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534403dcb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440208590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534403a5110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534402288d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c1710>, 9884001.042343099)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534401fa610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c0de0>, 9884000.85207798)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534403dec90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c1e80>, 9884001.254591027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440351350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c18d0>, 9884000.61686147)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534403a5250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440232710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a64a0>, 9884004.354836958)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440123f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400fbcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a7770>, 9884006.163621703)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440161c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534401a01d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c2c10>, 9884004.895099428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534401a0290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534405bd7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344000c7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344001b710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a5550>, 9884003.98406914)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405bdb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440182090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f0050>, 9884004.120627703)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344000cf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534401922e0>, 9884004.069601385)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440019610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c20b0>, 9884004.362319876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440182b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534401348d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440191a90>, 9884005.459517198)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405bd410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400672d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440088fa0>, 9884007.453446884)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440067890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400d9ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534400898d0>, 9884007.146252455)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400da150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400d87d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344008b000>, 9884007.601036064)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440045e90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344004de10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff638d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400aa010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a7c40>, 9884007.217737235)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400bd910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344008bc40>, 9884006.763661793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff63790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440088590>, 9884006.520387568)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400aa510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe22710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404cb9a0>, 9884009.927198783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fee6f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ffec5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c2270>, 9884008.588139564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ffec3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc2b410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd21b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd0f6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff3ec50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7f2a0>, 9884010.734775065)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd0d350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7fd90>, 9884011.356122443)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd91c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc58910>, 9884014.281071052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc2bfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fe660b0>, 9884011.211206544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd21d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb60390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7c600>, 9884014.979884332)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc74150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fcca410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc5bbd0>, 9884015.435474308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fcc90d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc5f450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5de10>, 9884015.704809852)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc5f150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc9f5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fcc11d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc594e0>, 9884013.538706308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc9f450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc59fd0>, 9884013.603914052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe81e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344013e550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5f070>, 9884016.428139515)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa07d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa76d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3a820>, 9884020.078333825)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fcc3190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa8fad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3b700>, 9884018.26762598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ffd0ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa62dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc9d190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941f230>, 9884016.1116952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa637d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5f230>, 9884016.141780844)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc9d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fbd2e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f932660>, 9884019.468257513)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fbd3010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f800bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82cc20>, 9884020.17605177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f958890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb97e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f9a07d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f9b6d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3b3f0>, 9884017.836785672)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f923290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f932040>, 9884018.55621153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f9a05d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f932820>, 9884019.135478688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f9b74d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f70fd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f931550>, 9884025.837690316)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f8141d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f73a850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f719010>, 9884022.946244936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f73ab90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f7b8c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f718a60>, 9884022.105285753)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f7b8bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f8c1790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f848f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f738690>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f75a8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f77ded0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82e9e0>, 9884021.965944314)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f8c1590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f930c20>, 9884021.059585562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd62ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd6a6d0>, 9884021.814388756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fcf15d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f7192b0>, 9884022.125406032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f75a510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f719b00>, 9884021.76962143)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f77e610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f8ed250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f823f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f639a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f8756d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd30f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff61450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534404ce8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534405be9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440839910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c61250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b95050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d77930>, 9884039.971498368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f8ec1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d74a60>, 9884044.918614788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f6136d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae3d20>, 9884041.90806224)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f638a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae25f0>, 9884040.537023196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f876910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962fbd0>, 9884042.538213598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd31690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8fd20>, 9884043.513493214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344002f750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a39320>, 9884048.605416942)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534404ce810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19aeb0>, 9884039.894742021)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405bd1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3af90>, 9884041.55649458)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344083ae90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c73bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c403d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bf8710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f64590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345924e490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345901e8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458333790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345942c1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a722d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e33f0>, 9884048.21957688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443ff8450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e19b0>, 9884040.214874027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f1c590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dce50>, 9884040.22712568)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bb1ecd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993cec0>, 9884048.277192432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c7f8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c75b0>, 9884040.893500226)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c416d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b6190>, 9884048.292802168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bfb0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208f7e0>, 9884040.925031794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f647d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadede80>, 9884041.046226555)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345904acd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3ee0>, 9884048.198261917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459016110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344067e1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440699d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344069bd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534406b9ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534406d5150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534406f1890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344061e510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440512d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344051add0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440528fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344054c090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440593790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534405c6f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344084b890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534405e0910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440538e10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440406e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344041a7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440846110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440441e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440464450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534405bc2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440473350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440318b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440352890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440368790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534403bc310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534402fc650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440310a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534403471d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440443250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440309d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440723310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440259750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344038a390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344028b1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440352a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440208bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440114dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440117790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344013e110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440186090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344027c890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534401ddad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440208f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344005cf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534401a3c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400a8e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff01790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344008c350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff21390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff2c0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff50890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff6dbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff9e710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff93750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff6cb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe03d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe146d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe30ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff2a910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe474d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe78450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400a9ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe90c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fea4650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fea6610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fecc650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ffeedd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd05250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd2d710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd63490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd84ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd8be50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fdc9e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fdea2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fdebf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc1e8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc27650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc31350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc52990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc5e450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc9c490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fcc2e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb27110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb74510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb8e990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fbac210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fbc06d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fce9710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb0ead0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb52590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fbe5010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa20950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa34990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa6d510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fbf15d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa9b910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fab76d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fad8910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fbc0050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f9218d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f938b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f9621d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f98c1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f9e2e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc279d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f8370d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f866110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f875e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f970790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f823510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f877d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f8865d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f8d9250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f7297d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f7aa510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f7c9350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f6fe090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f7a8b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3ba80>, 9884041.97674602)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c62990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a39e80>, 9884048.397893475)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ae37d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f3460>, 9883994.035598569)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344067e950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437f50>, 9883994.138980577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440699c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c9710>, 9883995.224813068)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344069b8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f0ec0>, 9883993.98077066)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534406b9cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f34d0>, 9883997.071694903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440615b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cb0e0>, 9883995.349097366)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534406f1750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f1cc0>, 9883997.17207062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344061d490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f00c0>, 9883995.38856411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440513450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f2900>, 9883995.7000849)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534406da910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb0130>, 9883995.529092057)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440528e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff00c0>, 9883997.450191695)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344054ead0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff3ee0>, 9883997.50662504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440593650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff2ac0>, 9883997.462308384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405c7650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f0750>, 9883998.194368452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405c7c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590da890>, 9883997.989328839)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405e0510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff2350>, 9883997.35525227)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405393d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff0ec0>, 9883997.272653207)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405b7450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c9940>, 9884013.756523268)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344041a690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458203e00>, 9883997.696998049)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440838910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582023c0>, 9883997.59757008)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440441c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f2ba0>, 9883997.267345864)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344045bb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458316a50>, 9883998.810795171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534406aa750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458534590>, 9884004.18983124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534404739d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404c9c50>, 9883999.47769104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405591d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404ca6d0>, 9884000.555283088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440352750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404caeb0>, 9884000.163911588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440368590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c03d0>, 9884000.352786068)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534403bc810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582029e0>, 9883999.758496013)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440308910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458200fa0>, 9884000.237067616)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405d6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cbd90>, 9883999.81253583)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440347090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f0fa0>, 9884001.55411604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440442d90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582018d0>, 9884000.656561064)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344030a5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff3150>, 9884002.158481924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534403476d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c2d60>, 9884002.191719407)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440259e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c3a10>, 9884002.294416623)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405bc4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c3c40>, 9884002.331568968)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344028aed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404cb850>, 9884002.380318392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440352610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404c9470>, 9884003.588450072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440208a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a6040>, 9884003.955787633)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405bd3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a6270>, 9884006.345637793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534401173d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a6d60>, 9884004.620619062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344013dd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404c8910>, 9884004.749479735)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440185e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440190d00>, 9884004.382439)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344027c390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404c8ad0>, 9884005.017703066)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534401de290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440190980>, 9884005.415696628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440209450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a5d30>, 9884006.762425948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440345550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440191780>, 9884004.533896867)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440399cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c0210>, 9884007.522507124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400ab290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344008a3c0>, 9884008.286357664)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff01250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440192eb0>, 9884007.423645915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344008c550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344008a6d0>, 9884007.921944609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff21110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9d1d0>, 9884007.247976406)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff2c790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344008b690>, 9884006.756239863)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff50690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a6ba0>, 9884008.734977685)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff6da10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7cc20>, 9884007.5698236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff9e5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7c9f0>, 9884008.322519172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff93e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440193e00>, 9884010.277869452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff6c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7d240>, 9884009.130027993)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe03c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7e970>, 9884012.66173602)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400b8050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7f4d0>, 9884009.826566743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe31210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7f700>, 9884009.383491628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ffb3b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7f8c0>, 9884009.17975742)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344008fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fe64590>, 9884010.727468101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe78950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fe649f0>, 9884010.171123082)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe6c510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440088280>, 9884008.992342912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe90a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440088f30>, 9884009.702918865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fea4450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fe656a0>, 9884010.603200693)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fea6390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fe65c50>, 9884009.841412276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fecc450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344008af90>, 9884012.261192894)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ffeee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fe66a50>, 9884011.945751874)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440267990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fe67310>, 9884010.321882002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd2de50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd683d0>, 9884012.7052579)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd63350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534401933f0>, 9884012.629867584)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd84f90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd69390>, 9884012.72778718)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd8bd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd69ef0>, 9884011.2645874)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fdca510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd6aac0>, 9884017.370919943)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fdea850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd6af20>, 9884012.581983617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fdb3190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd6b0e0>, 9884012.207235996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc1ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd6b380>, 9884015.122322405)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe47c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd6a270>, 9884012.653611472)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc31050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc581a0>, 9884012.707520736)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc525d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc584b0>, 9884012.860243171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400f9d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc59320>, 9884013.537564682)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc8b190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc58c20>, 9884015.58238356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fcc3050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd6bee0>, 9884014.978464443)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff3e190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5cd70>, 9884014.893663596)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb74c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5d240>, 9884018.401409868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb8ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5dc50>, 9884015.056488557)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fbac750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5e040>, 9884016.089815296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc28710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc5a6d0>, 9884014.138543798)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440149f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7fa80>, 9884014.776351554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb0e910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440088d70>, 9884014.789835384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb52450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5e740>, 9884020.132526696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fbe54d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5f8c0>, 9884016.563538648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa20790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5fe70>, 9884016.553870566)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa35050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa38c90>, 9884016.798382519)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb7db50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa39b00>, 9884017.22373718)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe30ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa39d30>, 9884017.483397428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa9b7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3a3c0>, 9884017.570038984)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fab7590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3a9e0>, 9884017.40613044)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc47cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3b460>, 9884019.746420177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fbc05d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3bbd0>, 9884017.455562256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f9216d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5f5b0>, 9884018.911911812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f938950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3acf0>, 9884018.955122646)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f962550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f931a20>, 9884019.383692307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f98c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f933150>, 9884019.76369467)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fdbc690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3a5f0>, 9884019.36211206)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc27e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3b9a0>, 9884020.126856862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f836c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82ce50>, 9884021.817483006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f859c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82d400>, 9884022.319626924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc5e4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f931780>, 9884019.344796851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f970710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3a660>, 9884019.448413841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f823c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7c830>, 9884020.984928576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f8778d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f930520>, 9884024.302122623)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f93ed10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82db00>, 9884022.227005811)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f8d9110>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f718590>, 9884022.413057905)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f729f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f71a7b0>, 9884022.68458851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f822590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5e7b0>, 9884024.250939928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f7c8f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f7198d0>, 9884022.281324957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f6fe190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f71a580>, 9884037.400303248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f7a9350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fdd97d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe31610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b78ad0>, 9884043.632934684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc30350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a0b0>, 9884044.203337416)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe31cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ffff150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344040b550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534588639a0>, 9884042.822039517)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344040bd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fea54d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534401b8250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458863150>, 9884042.96020394)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534406e6f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b79550>, 9884043.163899468)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534401b8350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f7e0e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b782f0>, 9884043.385586835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f739c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344025b910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344021db90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458862970>, 9884044.517341357)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440065c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a040>, 9884042.054833028)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344025a590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458860210>, 9884042.292845882)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440399090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534405c6990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407335d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747d90>, 9884045.033074358)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405c5990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440884650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745b00>, 9884047.356419057)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440887dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440761a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407a35d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747a80>, 9884046.831597025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440761590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747d20>, 9884046.117558748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407a3fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440884850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a5f60>, 9884063.559294233)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408868d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f823e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a82310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863e120>, 9884045.987705292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f822510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863fc40>, 9884050.36860101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a81c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344081a110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408fbf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863e580>, 9884047.84406601)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534409449d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404c8520>, 9884048.13471138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408f8ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fbd28d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd52510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440478a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408cd750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458744f30>, 9884048.022252906)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440643610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407480d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458746a50>, 9884044.989779511)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344074aed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863d710>, 9884048.52909774)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344094c3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407f1a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458744bb0>, 9884044.492964853)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd53610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534587458d0>, 9884044.225062508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344047b650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747540>, 9884044.755764384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344047b710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745f60>, 9884045.155814208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407f3510>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ae01d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863e820>, 9884048.984389788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ae0310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a41c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75550>, 9884049.250267887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a43190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408d9390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d76ba0>, 9884048.907442162)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344099ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bad610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d775b0>, 9884050.130026758)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440884050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440938390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d742f0>, 9884049.4091924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344093b550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b71e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b5bc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d741a0>, 9884051.203924263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b73190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae3ee0>, 9884050.79122804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ae9ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440839410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a16bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863ff50>, 9884050.63901169)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440838a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75ef0>, 9884050.774297856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a17550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bb3190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863e2e0>, 9884053.726756608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bb2dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440cccc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bbe6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408cd410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ae4f50>\n",
      "Evaluating workflow:  24%|██▍       | 120/500 [00:21<00:37, 10.09it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408387d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a7e250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ac7c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863d4e0>, 9884048.765129868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f6bc4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d749f0>, 9884048.968911042)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ac6e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae0600>, 9884051.461225696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440cccb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a580>, 9884052.815636609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b7da90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863cc90>, 9884046.405262213)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440826450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863fcb0>, 9884047.53863666)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ae5850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b78980>, 9884047.645692697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a7fa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440600d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a387c0>, 9884050.179035628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440850a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440dbfe90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a6d0>, 9884052.549407093)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440dbe050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440dbb590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534587472a0>, 9884050.62426473)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440dbb650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440eca650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ed7750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3aa50>, 9884052.186217448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ec9710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a38ec0>, 9884051.894039787)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ed7410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c990d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e32790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459301e10>, 9884052.530313272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344000d0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d758d0>, 9884051.379174894)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e33cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344081b390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b580c0>, 9884052.755410368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344081ad10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440da12d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae3b60>, 9884056.240540847)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a03f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c9ba90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b59da0>, 9884052.555953125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c29a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440fc8450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b5b1c0>, 9884052.694783213)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440fc85d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bf9bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d00850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c48310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c7ddd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c838d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a38910>, 9884050.798434228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440643e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3af20>, 9884051.010172255)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c7cad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f3e00>, 9884054.611521434)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bf9b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f1080>, 9884052.684692776)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d00610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3ac10>, 9884051.539837647)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c48c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d8d650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f22e0>, 9884060.47811204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d8fdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b29750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a2e0>, 9884054.500332305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b2b9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441024250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75630>, 9884056.344181076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534593c0610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440dfa3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d4e0>, 9884053.736368384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440dfaf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410698d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d470>, 9884054.48206306)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441068050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f8c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d8d0>, 9884055.233931623)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f8edd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f729d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f94c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f05b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d630>, 9884054.712644158)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f948d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f620>, 9884056.929698467)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f06950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440fe8190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459519c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae18f30>, 9884058.6741725)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459519890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441064110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19400>, 9884054.669459945)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f2ba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd7d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441036190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bfce90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d43950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e0a090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5e270>, 9884054.591859214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bed350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745860>, 9884053.487251772)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bfd750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3b150>, 9884055.011051333)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440beee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d240>, 9884053.602749849)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e09c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae1a890>, 9884054.483840838)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441000250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae1ad60>, 9884055.944333749)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441037590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f1710>, 9884052.660416933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d41210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344104db50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19b70>, 9884057.692304848)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344104d6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ec2490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345912cfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae1aac0>, 9884060.464794053)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534409b0950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae18590>, 9884056.807688735)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345912c450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344109e750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030af90>, 9884058.565032247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344109e3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345919d790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e25290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f1b70>, 9884057.758096581)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441081090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3ae40>, 9884060.759822747)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e27790>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458efd250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534592fb110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e8b690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f18b40>, 9884056.297834411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440fb2990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030b7e0>, 9884058.136927208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e88d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f9d3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f186e0>, 9884058.719084635)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f9da10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458fda310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b28d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030aeb0>, 9884057.04984336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f6c7850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344118f110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75b00>, 9884058.971396772)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534591bcad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410049d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345aa122e0>, 9884056.644756887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458efe8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a477e0>, 9884058.433813548)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534597a9c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f18210>, 9884059.643133046)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344118f490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19ef0>, 9884055.684915096)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441005410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534411caf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a46d60>, 9884059.585167006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534411c9590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534411a2e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a47380>, 9884058.943359653)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534411a1310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ed1990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b78bb0>, 9884058.947906006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ed1e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b92e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458adc9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb37e0>, 9884060.560180482)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458add9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534588d8390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458bab110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5da20>, 9884061.959282177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458bab390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a9c850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458951650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb3af0>, 9884062.312409094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458952f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345885ba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534588383d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458efc910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459080bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345815fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458cccc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb3d90>, 9884059.888152324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458bc4210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5dda0>, 9884061.1801799)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534588d8490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a458d0>, 9884059.102900555)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534595c2010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5ca60>, 9884061.371918876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a9dd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a446e0>, 9884058.921277076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459083b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb3700>, 9884060.500735251)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b0b990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1080>, 9884061.147463009)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458838950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030a430>, 9884058.332376117)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458efc9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a47cb0>, 9884059.354007248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458cccbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534591703d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b1a20>, 9884063.367817033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459170210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534591b0fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458360b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a475b0>, 9884063.348855382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459293a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a46ba0>, 9884066.061389674)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458360f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534588b3d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863def0>, 9884063.330526948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534588b07d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a6f310>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0c20>, 9884061.408022048)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a6d010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a4c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0de0>, 9884061.506748293)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458616c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458951b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534581b7a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a44590>, 9884063.055263199)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589b7810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae1b380>, 9884062.576770524)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581b4190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410ba110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b00c0>, 9884063.499683265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e92950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534584c5b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587ebc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e0f30>, 9884063.462983817)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458221510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e0bb0>, 9884063.643182367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345888d150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458da0d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458da3210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534595880d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534588ef1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b1a90>, 9884061.644987324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a44310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb07c0>, 9884062.93866208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458da1cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f19a20>, 9884060.961553484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458fa4f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b2190>, 9884061.879322512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534588ef150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459e41890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b1240>, 9884065.422354417)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345f59b910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459aca3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e2270>, 9884064.072236732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459435ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443fc9550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458450150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3230>, 9884063.923759984)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441780e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e2c10>, 9884064.528589794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458450990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fad46fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e1320>, 9884064.045822743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fad45fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d49bc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5fe70>, 9884065.271674763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d498490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458331b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5d940>, 9884064.756271992)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458330150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459f63fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5fa80>, 9884065.2778588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459cc9510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459d9be10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d22250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443fe6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b3540>, 9884069.73691758)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534583ae750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5db70>, 9884065.25297356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585b3190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a600750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0c90>, 9884065.821402865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a3e5850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458128610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459634950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f8c790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345838dcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae18c20>, 9884065.0310743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459d9ae90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5d160>, 9884065.7959922)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443fc8fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5cc20>, 9884066.720318291)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459637290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb0ad0>, 9884063.097999904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f8d650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e2580>, 9884063.42134424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345838ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f8e350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5ea50>, 9884068.268898856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534598f44d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345960b310>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b2cf0>, 9884066.809721164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459609bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d327e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f27b0>, 9884066.55383153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3263d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a6efa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff3230>, 9884067.215575833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a6ee610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459683f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2660>, 9884067.776710376)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a40e010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459d986d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443fcba90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e37e0>, 9884067.42878011)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459d9ad90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2f20>, 9884066.801023645)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fadbc4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d56de50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b3d90>, 9884068.98861554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d70f690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345f65ff50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5ed60>, 9884067.691721275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345f65cc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a60ecd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d4d7e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3d90>, 9884067.975239124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ec2c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443e0d5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5e430>, 9884068.437963834)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345ad7d490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d6f7d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a68b7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ea1e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a852d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1010>, 9884068.738098308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d4d6750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2f90>, 9884065.816976996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a8524d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941e3c0>, 9884068.585100943)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d6f7ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1e80>, 9884066.188614141)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459677550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a463c0>, 9884065.752446298)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ea0890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d534350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941d6a0>, 9884067.783382308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534411c8dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a810a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941c2f0>, 9884070.616848009)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a066090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345ad27950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941fa80>, 9884068.397087775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a600790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e4ea50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fade6910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5c3d0>, 9884069.69571876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e4c210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f0910>, 9884070.274614217)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fade4910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345f66c590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443113050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941ca60>, 9884071.113540852)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443110ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a670910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d9630>, 9884070.605437735)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a6738d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f1ca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590db4d0>, 9884071.085922591)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584aca90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bcac390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d9e10>, 9884073.577165226)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bcaf950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbc9790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc16a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d8600>, 9884069.834165294)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d55f450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590db150>, 9884070.1881329)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc16710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc25510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583efcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d581a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5c8a0>, 9884071.045061488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345f66e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941f770>, 9884068.507405704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d580e50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590ca270>, 9884070.284294892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc27a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941c7c0>, 9884067.91974915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534583ee710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a265f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f3380>, 9884071.457695032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a265dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a4cf350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585172d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2890>, 9884072.390906492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a751050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590ca900>, 9884071.29269405)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458514ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458630bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc2c80>, 9884073.57149553)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458633c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345869a350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0600>, 9884071.838181453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345869ad90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458800a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc1240>, 9884077.442622684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534588011d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f990d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590da120>, 9884072.968859527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc244d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534430b0a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458924890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9f690>, 9884072.896125732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430b22d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9fb60>, 9884072.419690756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589254d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a112290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a2f38d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443061e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345901f150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443057710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a073a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1e10>, 9884070.65575398)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a072310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c98d0>, 9884070.691495717)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345901e910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c9e80>, 9884072.874300389)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a110f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d9a20>, 9884070.48979056)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3ceb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c87c0>, 9884070.59028738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443060890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3a80>, 9884070.957655732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443085b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a158f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9cd70>, 9884074.790964426)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a159610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585a3210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9ca60>, 9884074.29446544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d60fe10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585334d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9d470>, 9884074.882953843)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ebe950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441846950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9f380>, 9884074.734507976)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441846810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f06510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419d6c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b55630>, 9884074.116525756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ab3750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b54e50>, 9884076.1194138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534419d6cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441890490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b56190>, 9884073.608432952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441890350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441701650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b542f0>, 9884073.678197479)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441701610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431f7d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b56820>, 9884078.000853239)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534431f6290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344195d990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b54ec0>, 9884074.395300036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344195dd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344194a190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b54f30>, 9884075.970418848)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441733190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441915bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b54210>, 9884074.867253903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441915710>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441aed9d0>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      " {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.28571428571428575, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419035d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbc6f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442ed8910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431091d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c8de0>, 9884073.244033743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ed9ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941d860>, 9884073.289432555)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430570d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b56ba0>, 9884075.382321933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441aed650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b54600>, 9884075.202725036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534586c83d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9c2f0>, 9884072.817978663)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbc4d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441973c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458316c10>, 9884075.26013316)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441886b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534418f1cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b55400>, 9884076.127162669)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534418f3790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534418e87d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0e50>, 9884077.260308104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534418ea4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419a0e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441aba050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458317070>, 9884076.845395574)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534419a0d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458315400>, 9884078.21851227)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ab9410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534418b7a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458317460>, 9884076.337204237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534418b4710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a75210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314fa0>, 9884077.964436913)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a76690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a484d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896e9e0>, 9884076.513830708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441949650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fad2d450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9fe00>, 9884076.791536875)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441bc8350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441aa0b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b96210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b569e0>, 9884076.599892313)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a59710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1320>, 9884076.88098704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b96ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441bf3ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b5b010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896fe70>, 9884081.31024997)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a073610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896ec80>, 9884076.998539226)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b5a910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441cef3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441cc7490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419e8790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896f380>, 9884080.86853824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ceefd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458317e00>, 9884076.007909872)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534419ea2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d8f30>, 9884077.485643491)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441cc6f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a4ce110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc2890>, 9884079.087167984)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345f66e950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344171bd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896d710>, 9884082.081866875)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441718550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441def2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a70590>, 9884078.587863417)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441dedfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ce46d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458316ac0>, 9884079.504368559)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a33190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d9e7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d59990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a71940>, 9884080.994349632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441bafa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a73380>, 9884079.718769047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c50410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c284d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a246690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896c0c0>, 9884081.528772805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c29cd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd6b7e0>, 9884078.940092811)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a244210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d03290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d77650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536d60>, 9884080.884531453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d77b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345ad24190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441e76990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc1ef0>, 9884080.04753352)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441abafd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585360b0>, 9884080.644364253)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e76090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c354d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b1a210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b7a250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a73930>, 9884093.930982368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ec9990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a70360>, 9884078.098604882)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c37b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a708a0>, 9884078.256475592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b19190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a73bd0>, 9884078.456202816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b794d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d13390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585356a0>, 9884081.408171214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d13d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f98a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437070>, 9884081.855125688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f98f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f6f5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f70190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536660>, 9884080.704204302)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441dff9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458436f20>, 9884081.459424617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f72910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c1e250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896f310>, 9884083.025561055)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f885d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534420e08d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437310>, 9884081.332871504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534420e0190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a30fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435f60>, 9884082.851332562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441bafa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442089890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584367b0>, 9884083.140028048)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534420893d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441fc5d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458534ad0>, 9884082.718906965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441fb4e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442000350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585357f0>, 9884083.067270985)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442003750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421df7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441e34950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b97050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441fe74d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441feecd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a70e50>, 9884080.731191382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441fef950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b56b30>, 9884080.99312478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d329d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896ea50>, 9884083.532136474)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534421de050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc14e0>, 9884080.123141188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441dca6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536c10>, 9884080.383900004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441fe40d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f29050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437150>, 9884087.603528876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f28350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344216add0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8f3f0>, 9884085.144748284)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344216aad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ea31d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8d710>, 9884084.6732642)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ea3ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421c6f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8d550>, 9884084.237788664)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534421c5b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ab9750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344227afd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534583165f0>, 9884084.154294042)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ab9e90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442263110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e0b0>, 9884085.342079721)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442261f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442292150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442217a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e190>, 9884089.13656738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442292cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8c1a0>, 9884085.114253452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534422171d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442203790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513460>, 9884084.155545335)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442201b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a06ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459511710>, 9884084.28054042)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344206b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344238a610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344234bb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8f700>, 9884085.347116677)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344234bfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534423ec3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437620>, 9884089.094607612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442313750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a8bf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8c130>, 9884085.382828906)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534422799d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536430>, 9884085.067826852)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344238b490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8f690>, 9884083.420889292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534421685d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8cc20>, 9884086.864361422)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534423ecf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344217acd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8db00>, 9884083.000791537)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442179d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442307850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459511d30>, 9884087.412329324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442305350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d437a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459512510>, 9884085.7575324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344207e910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344229ed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442457e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513540>, 9884085.821586542)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442131e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459510d00>, 9884087.705763808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442455b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534424b4ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442401210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962eeb0>, 9884086.157210376)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534424b5e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c8a0>, 9884087.875974577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442402f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425bddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c0c0>, 9884087.070740504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344247aed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425544d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962f700>, 9884088.085287644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442159c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442255550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962d240>, 9884088.009878604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442254350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442536b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c3d0>, 9884087.854804972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442534d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344257de50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426c1250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962f620>, 9884088.669555556)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442132ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962e120>, 9884088.197611134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426c3ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344234be50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344246b650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a04b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584373f0>, 9884085.777533334)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441cec290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738050>, 9884089.345066644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344234bc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c590>, 9884086.319114449)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344200f110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e8f350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738280>, 9884089.768869055)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e8de90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442569690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534597394e0>, 9884089.677076038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442568190>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426f0790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426dd150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425d0a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459510210>, 9884090.319607897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534425d0c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344274e790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458434c20>, 9884090.063808246)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344274eb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344234a710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427461d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993d550>, 9884089.959812108)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d3dcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993eac0>, 9884091.05235393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442744150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427165d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459510440>, 9884091.67562304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442716a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442cd0bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427384d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993da20>, 9884094.388244988)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442cd1350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993f230>, 9884090.390178343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441dfdf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344241dfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442fd6c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425850d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442638050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534597391d0>, 9884089.3016037)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426f3390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a820>, 9884091.14699885)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427d8b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738c90>, 9884088.760439098)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442585590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962de80>, 9884091.151225723)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344241c750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738ec0>, 9884088.518487062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442fd6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738b40>, 9884089.0898536)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344263a950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344279ff50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442890dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993f380>, 9884091.307670947)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344279d290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534597396a0>, 9884092.033698944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442891310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344231e150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458534c20>, 9884092.22613086)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344231f110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442313390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5db00>, 9884091.429306045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442313010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427f6ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513700>, 9884093.300096257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427f4610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427a2410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459739be0>, 9884092.507392982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427a3e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442fb5990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429caa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5f0e0>, 9884091.965511614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e8ecd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5d6a0>, 9884092.58428779)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429ca810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429f5750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429e2890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993fa10>, 9884091.716885071)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429f4650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5f8c0>, 9884093.228618275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429e0c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e069d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c050>, 9884092.674236916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e06d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442971b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534428bb010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d9b7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429e4e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993c130>, 9884091.390826782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428b8310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd6ae40>, 9884094.296845954)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442972910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962cd70>, 9884092.09835106)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d99310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5de10>, 9884091.781263204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429e6c10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442790a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a199a20>, 9884093.967874605)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442984190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e25d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a1997f0>, 9884093.777137464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e25ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534428a4bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442924310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19acf0>, 9884096.045862537)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428a6550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19a0b0>, 9884095.26705018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429245d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344294b010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8d00>, 9884094.011301436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442948ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429ca410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344097d910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5edd0>, 9884094.693633456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429cb450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cbf50>, 9884095.08191204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344097f210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d36ec90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ca5f0>, 9884096.276096076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d36e150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407e16d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c9b00>, 9884094.311659576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d30e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f552490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f56c290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513ee0>, 9884094.463914612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f552210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ca430>, 9884094.585096028)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f563a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f571b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a1992b0>, 9884095.34088196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f5719d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442eee3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e941d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344296b490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19a3c0>, 9884093.348844511)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e96dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a198520>, 9884093.31757886)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c95e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a1986e0>, 9884093.197183538)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442968ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f597890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c9010>, 9884095.335794356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f597ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f583490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f5c4a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5db70>, 9884096.943103852)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428aefd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e771c0>, 9884097.952508094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f5bb190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442dd1f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cb3f0>, 9884096.915235309)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442dd0250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f5521d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f434e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5cc90>, 9884097.2712121)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f551e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19ac80>, 9884096.07741938)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4349d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f43e0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5c830>, 9884096.953265484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f43ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442972690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f461010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e77e00>, 9884097.251668079)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429722d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c91d0>, 9884096.468666447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f461750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f470890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e77d20>, 9884096.63756074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f470690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f414bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e758d0>, 9884097.091921052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c3c110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f492750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4ada90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e74830>, 9884097.158482453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4929d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4c0050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e26ad0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f5cf150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cb4d0>, 9884096.977706289)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4adf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5f310>, 9884098.00710833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4c0350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cab30>, 9884095.1176043)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e24990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e74bb0>, 9884095.217885492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f5cef50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f572d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7000>, 9884098.158755379)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f572dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4f3850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f30a410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b5b70>, 9884097.36732457)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4e5010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7ee0>, 9884097.916124737)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f30a210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f335350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f33e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e77850>, 9884098.503253313)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f33c0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f34ba10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19bf50>, 9884098.652239995)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f34bf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f37c050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9e5f0>, 9884099.340738563)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f37c550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f39c710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2fef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f3bf490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7460>, 9884099.721218916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4361d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ca4a0>, 9884099.76913121)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f3bf350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f3c6d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9cde0>, 9884099.796274064)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f3c6c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4e1a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5cc20>, 9884100.07934263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1fc790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4e0c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f5cfa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9e120>, 9884100.59686302)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f335150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7e70>, 9884099.499721268)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f39c5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b5c50>, 9884097.690682497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4e0a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c6e0>, 9884099.21848748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f3d37d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b60b0>, 9884097.481451828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f5cf690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f3b0910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2204d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d70d00>, 9884102.758419923)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f3b0990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5e890>, 9884100.262377094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f220b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f238210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e76e40>, 9884102.523688888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f238710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f244d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d707c0>, 9884100.91845899)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f244b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2574d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f26ef50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9fd90>, 9884102.70165354)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f257b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19b380>, 9884100.001659237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f26ed50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f3e0390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2904d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d709f0>, 9884103.444979075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f3b31d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f296450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d73a80>, 9884101.451603804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f296310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2ad850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d71b70>, 9884101.11105798)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f2ad710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2afa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8d70>, 9884101.712791933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f2afd50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2edf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f0fb710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e75550>, 9884101.748441048)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f0fb5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f11ca50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2068d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b4520>, 9884100.386589617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f290bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c9f0>, 9884101.760667011)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f2ee710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8f070>, 9884101.455887971)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f0fba90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d72040>, 9884099.743705776)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f206e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f13fd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f143810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8d940>, 9884103.074276512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f13fbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8d780>, 9884102.751491113)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1436d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2b9990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8e200>, 9884101.796979228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f207650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f128c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f17c450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8f310>, 9884114.114075424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4444d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8f380>, 9884103.215965897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f177b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f101fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a0980>, 9884102.737137623)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f101cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f168c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1ae1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a1f60>, 9884104.08220599)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f196610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a1240>, 9884103.226316843)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1ae890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1d5210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f11c5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8f5b0>, 9884103.758206405)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f11c410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2ef6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1e1b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a0fa0>, 9884103.767166227)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f2ee510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a31c0>, 9884103.902799746)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1d7990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1bf110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f031a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a0670>, 9884104.537429707)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1bfd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8cbb0>, 9884104.354817787)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f479f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f03d190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8dfd0>, 9884104.193024091)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f047510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f26ef10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8f000>, 9884105.210636022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1d4e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8910>, 9884104.285589669)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f03d910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8e430>, 9884101.397942036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f26eb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f05cdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a1d30>, 9884104.957881855)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f053190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f09d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8c2f0>, 9884105.644007623)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f09e690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f3b3290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f0d0f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6ff50>, 9884105.5832572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f3b2d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6e740>, 9884105.031790173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f0d0b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f079110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6cc90>, 9884105.997831784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f079250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f14c950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6fe70>, 9884105.166705614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f14ccd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eeff7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef14390>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6d9b0>, 9884106.095233032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eeffa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6c0c0>, 9884105.390731508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef14a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef2c1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6f2a0>, 9884105.708976595)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef2c6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef4c3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a1160>, 9884106.197479373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef39c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eefe110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c5e80>, 9884106.271943973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eefeed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1be110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef74750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f3e06d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c7a80>, 9884107.217103297)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef61390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c5390>, 9884107.442606805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef74dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8d860>, 9884104.564323476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f18d610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef7fd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c52b0>, 9884108.310752615)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef7fc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef31c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c7b60>, 9884109.059097562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef31990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef9c8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343efb7c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c4440>, 9884106.81373936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef9fcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c62e0>, 9884107.499061607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343efb7b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef7d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c74d0>, 9884107.744180659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343efbda50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ebfb250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c54e0>, 9884109.294226073)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ebfb110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f244d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6e200>, 9884121.458035177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1ae190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec16150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec25050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed5a20>, 9884107.601272443)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec164d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed7690>, 9884107.460709153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec25710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343efe1ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed7d20>, 9884107.375734512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343efe1fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec149d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed5cc0>, 9884108.698506117)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec15210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f05f350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec79650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6f310>, 9884108.39952349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f19ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed6430>, 9884108.176443731)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec79d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec3bc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed4b40>, 9884108.473888302)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec3bc90>\n",
      "Evaluating workflow:  24%|██▍       | 122/500 [00:21<00:53,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 124/500 [00:22<01:29,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7777777777777778, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 127/500 [00:23<01:13,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 130/500 [00:23<01:08,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 132/500 [00:23<00:51,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 134/500 [00:24<00:49,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 136/500 [00:24<00:50,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 139/500 [00:24<00:48,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 143/500 [00:25<00:47,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.923076923076923, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 144/500 [00:25<00:52,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 145/500 [00:25<01:13,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.039999999999999994, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 149/500 [00:26<00:45,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.42857142857142855, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 151/500 [00:26<00:43,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 154/500 [00:26<00:49,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 157/500 [00:27<01:02,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2666666666666667, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 159/500 [00:28<01:08,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 164/500 [00:28<00:39,  8.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 166/500 [00:28<00:34,  9.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 168/500 [00:29<00:49,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 170/500 [00:29<00:46,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.625, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 172/500 [00:30<01:36,  3.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.1904761904761905, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 173/500 [00:31<01:29,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 177/500 [00:31<00:58,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 183/500 [00:31<00:35,  8.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 185/500 [00:32<00:46,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.09523809523809525, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.23076923076923078, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 187/500 [00:32<00:43,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.28571428571428575, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 189/500 [00:32<00:42,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 190/500 [00:33<00:47,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 192/500 [00:33<00:56,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 193/500 [00:33<00:53,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 194/500 [00:34<01:21,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 199/500 [00:34<00:50,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 203/500 [00:35<00:40,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.7777777777777778, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 207/500 [00:35<00:25, 11.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 211/500 [00:36<00:42,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 215/500 [00:37<00:39,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 217/500 [00:37<01:00,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 218/500 [00:38<01:09,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 221/500 [00:38<00:50,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 222/500 [00:38<00:54,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 223/500 [00:39<01:00,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.21428571428571425, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 228/500 [00:39<00:42,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.15384615384615385, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 231/500 [00:39<00:32,  8.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 235/500 [00:40<00:27,  9.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 237/500 [00:40<00:34,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 238/500 [00:41<00:56,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 239/500 [00:41<01:14,  3.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 240/500 [00:42<01:11,  3.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 242/500 [00:42<00:58,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 247/500 [00:42<00:31,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 248/500 [00:43<00:33,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 251/500 [00:43<00:26,  9.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2666666666666667, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 253/500 [00:43<00:43,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 256/500 [00:44<00:36,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 257/500 [00:44<00:42,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 260/500 [00:44<00:34,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.13333333333333336, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 263/500 [00:45<00:32,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 265/500 [00:45<00:38,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.7499999999999999, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 267/500 [00:46<00:38,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 271/500 [00:46<00:33,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 273/500 [00:46<00:29,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.22222222222222224, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 276/500 [00:47<00:25,  8.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 280/500 [00:47<00:25,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5142857142857142, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 283/500 [00:48<00:33,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 285/500 [00:48<00:28,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 286/500 [00:48<00:37,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 290/500 [00:49<00:28,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 292/500 [00:49<00:39,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 295/500 [00:50<00:29,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.23529411764705882, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 298/500 [00:50<00:24,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.9090909090909091, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.012903225806451615, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 300/500 [00:50<00:28,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 304/500 [00:51<00:25,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 305/500 [00:51<00:27,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 308/500 [00:52<00:28,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 311/500 [00:53<00:48,  3.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 316/500 [00:53<00:28,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3571428571428571, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.7058823529411764, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 322/500 [00:54<00:24,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 326/500 [00:55<00:23,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 329/500 [00:55<00:22,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 330/500 [00:55<00:30,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 334/500 [00:56<00:23,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 336/500 [00:56<00:18,  8.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3157894736842105, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 340/500 [00:57<00:22,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 344/500 [00:57<00:15,  9.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 348/500 [00:58<00:29,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 350/500 [00:58<00:23,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 355/500 [00:59<00:15,  9.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 357/500 [00:59<00:15,  9.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.047619047619047616, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 359/500 [00:59<00:18,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 361/500 [01:00<00:19,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 363/500 [01:00<00:27,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.21052631578947367, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 366/500 [01:01<00:21,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 369/500 [01:01<00:21,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 371/500 [01:01<00:15,  8.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6363636363636364, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 374/500 [01:02<00:19,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 376/500 [01:02<00:18,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 379/500 [01:03<00:21,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 384/500 [01:04<00:14,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 386/500 [01:04<00:15,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 387/500 [01:04<00:15,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.26666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 389/500 [01:04<00:17,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 391/500 [01:05<00:21,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 395/500 [01:05<00:12,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 397/500 [01:06<00:15,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 398/500 [01:06<00:20,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.058823529411764705, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 401/500 [01:07<00:17,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 403/500 [01:07<00:16,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 406/500 [01:07<00:12,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 410/500 [01:08<00:11,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 411/500 [01:08<00:11,  8.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 412/500 [01:08<00:17,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 414/500 [01:09<00:20,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 417/500 [01:09<00:15,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 420/500 [01:10<00:13,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 423/500 [01:10<00:10,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 425/500 [01:10<00:07,  9.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 427/500 [01:11<00:08,  8.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 430/500 [01:11<00:09,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 431/500 [01:11<00:11,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 432/500 [01:12<00:14,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 434/500 [01:12<00:11,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 435/500 [01:12<00:13,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4210526315789474, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 438/500 [01:13<00:09,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 441/500 [01:13<00:08,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8235294117647058, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 442/500 [01:13<00:11,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 443/500 [01:14<00:12,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 447/500 [01:14<00:09,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.22222222222222218, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 450/500 [01:14<00:06,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 452/500 [01:15<00:06,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 453/500 [01:15<00:09,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 454/500 [01:16<00:10,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 457/500 [01:16<00:07,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 458/500 [01:16<00:07,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 461/500 [01:17<00:06,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.47619047619047616, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 462/500 [01:17<00:07,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 466/500 [01:17<00:04,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.47619047619047616, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 468/500 [01:18<00:05,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 472/500 [01:18<00:03,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 474/500 [01:18<00:03,  8.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 475/500 [01:19<00:06,  3.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 481/500 [01:20<00:03,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 482/500 [01:20<00:03,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 484/500 [01:21<00:03,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 486/500 [01:21<00:02,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 488/500 [01:21<00:02,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 490/500 [01:22<00:01,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 494/500 [01:22<00:00,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 496/500 [01:23<00:00,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 498/500 [01:24<00:00,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.375, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 499/500 [01:26<00:00,  1.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 500/500 [01:31<00:00,  5.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.04347826086956522, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 5/500 [00:02<02:48,  2.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 11/500 [00:02<01:09,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 16/500 [00:03<01:05,  7.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 20/500 [00:04<01:27,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 23/500 [00:04<01:11,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 25/500 [00:05<01:27,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 28/500 [00:05<01:16,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 29/500 [00:05<01:25,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7272727272727273, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 30/500 [00:06<01:56,  4.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 33/500 [00:06<01:36,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 34/500 [00:07<02:25,  3.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 36/500 [00:08<02:03,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 40/500 [00:08<01:01,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.1904761904761905, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.17391304347826086, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 42/500 [00:08<01:06,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 44/500 [00:09<01:27,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 46/500 [00:09<01:24,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 51/500 [00:10<00:58,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 53/500 [00:10<01:01,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.1739130434782609, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 54/500 [00:10<01:42,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 57/500 [00:11<01:21,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 59/500 [00:11<01:17,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.125, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 62/500 [00:12<01:34,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 65/500 [00:12<01:01,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 67/500 [00:12<00:51,  8.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 69/500 [00:13<01:01,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 71/500 [00:13<01:03,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.30769230769230765, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 73/500 [00:13<01:03,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 74/500 [00:14<01:09,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 75/500 [00:14<01:18,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 77/500 [00:14<01:26,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 79/500 [00:15<01:17,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 80/500 [00:15<01:11,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 82/500 [00:15<00:58,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 84/500 [00:16<01:21,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 85/500 [00:16<01:28,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 87/500 [00:16<01:29,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 90/500 [00:17<01:24,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 91/500 [00:17<01:21,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 92/500 [00:17<01:28,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 96/500 [00:18<00:58,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 98/500 [00:18<00:46,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.28571428571428575, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 100/500 [00:18<00:54,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.28571428571428575, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 102/500 [00:19<01:35,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 103/500 [00:19<01:22,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 105/500 [00:20<01:08,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.1904761904761905, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 107/500 [00:20<00:59,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4615384615384615, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 111/500 [00:20<00:50,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 114/500 [00:21<00:42,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.16666666666666669, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 119/500 [00:21<00:28, 13.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4615384615384615, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 121/500 [00:21<00:46,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.9090909090909091, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 126/500 [00:22<00:54,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.7368421052631579, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 128/500 [00:23<00:53,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 130/500 [00:23<01:08,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 136/500 [00:24<00:51,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.21052631578947367, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 138/500 [00:24<00:43,  8.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 140/500 [00:25<01:09,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 143/500 [00:25<01:04,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.21052631578947367, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 148/500 [00:26<00:36,  9.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 150/500 [00:26<00:52,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 152/500 [00:26<00:43,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7692307692307693, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 154/500 [00:26<00:44,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 156/500 [00:28<01:44,  3.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 158/500 [00:28<01:25,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 162/500 [00:29<01:05,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.17391304347826084, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 166/500 [00:29<00:45,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.7142857142857143, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 167/500 [00:29<00:46,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 168/500 [00:30<01:50,  3.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 171/500 [00:31<01:27,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 175/500 [00:31<01:01,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 177/500 [00:32<00:50,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.42857142857142855, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 179/500 [00:32<00:55,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.08333333333333334, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 180/500 [00:32<01:12,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 181/500 [00:33<01:19,  4.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 182/500 [00:33<01:40,  3.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 185/500 [00:34<01:12,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 188/500 [00:34<00:57,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 189/500 [00:35<01:03,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 192/500 [00:35<00:57,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.07407407407407408, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 193/500 [00:36<01:22,  3.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 194/500 [00:36<01:30,  3.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 196/500 [00:36<01:09,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 198/500 [00:37<01:02,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 200/500 [00:37<01:02,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.0625, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 202/500 [00:37<00:49,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 203/500 [00:38<00:47,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 206/500 [00:38<00:40,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 208/500 [00:38<00:38,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 210/500 [00:39<00:54,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 211/500 [00:39<01:08,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 214/500 [00:40<00:52,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7777777777777778, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 217/500 [00:40<00:34,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 220/500 [00:40<00:47,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 221/500 [00:41<00:53,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 222/500 [00:41<00:58,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.15384615384615385, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.21428571428571425, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 226/500 [00:41<00:38,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 228/500 [00:42<00:53,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 229/500 [00:43<01:23,  3.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 234/500 [00:43<00:42,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 238/500 [00:44<00:35,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 241/500 [00:44<00:29,  8.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 244/500 [00:45<00:55,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 247/500 [00:45<00:44,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 249/500 [00:46<00:38,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.14285714285714285, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 252/500 [00:46<00:33,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 253/500 [00:46<00:36,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 255/500 [00:47<00:35,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 258/500 [00:47<00:42,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.1, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 259/500 [00:47<00:38,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 261/500 [00:48<00:40,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 263/500 [00:48<00:44,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 265/500 [00:48<00:35,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 266/500 [00:48<00:33,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 267/500 [00:49<00:39,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 269/500 [00:49<00:43,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 270/500 [00:49<00:49,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f01ec90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c7d90>, 9884109.920343217)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef61290>\n",
      "Evaluating workflow:  54%|█████▍    | 271/500 [00:50<01:11,  3.20it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eb73490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadeeac0>, 9884111.623940656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eb73f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eb66350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533faded2b0>, 9884112.407826945)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec48250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eb73ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadeec80>, 9884114.091456752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef31c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea23c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8027b0>, 9884115.66858478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ea23f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea65fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8020b0>, 9884113.203652415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ea655d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e9a2910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f1be0>, 9884114.563686844)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e9a2c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eafbd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2270>, 9884116.524652308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e9b5790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e9b0150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df620>, 9884115.146816468)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e9b0910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e928f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eafbf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea22d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e9bf2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f1860>, 9884114.856065264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f161c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2350>, 9884113.75427996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e92a490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f0fa0>, 9884115.218593892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ea228d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df380>, 9884115.25268576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e9bf0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e815bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a801550>, 9884119.00114484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e816890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e716410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d73310>, 9884117.767140549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e716750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e8b6110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a090e50>, 9884119.710059203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec5ed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e709d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9322e0>, 9884119.615831794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e709610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e856e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e95b150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4deac0>, 9884117.162096111)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e856bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a091780>, 9884117.093401572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e95b210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eca9410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef6dd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f49be10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4e42d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a3a2010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442783190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e7ac0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e7af510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e7da210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863c210>, 9884130.993713852)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eca9990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344008a970>, 9884131.678294372)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef6cdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896fee0>, 9884131.287139831)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344273be10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dd550>, 9884119.617958903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e7ac510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a932740>, 9884118.998961236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e7af3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4de510>, 9884128.813472388)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e7da010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c34c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426f2b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344231c2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344237f350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ebc110>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a19d7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410c7f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534405911d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe7bf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421053d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a007350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3fa350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a350>, 9884130.896003656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458dc0b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa60c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f773510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863c600>, 9884133.417472292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429e55d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae0210>, 9884131.665916858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442da3850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896c8a0>, 9884130.795691747)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442231b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745470>, 9884131.100314492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442202010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c3230>, 9884135.798012955)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ebf750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534588603d0>, 9884136.790014423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443e0c3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458863ee0>, 9884135.910170903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440eb70d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a270>, 9884131.293220501)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440589e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a3c0>, 9884132.38738943)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fda9650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b79940>, 9884131.865575893)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458166250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404cb230>, 9884130.892245283)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3f9a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9d400>, 9884131.290388156)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc76010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440591d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6cde0>, 9884133.522347543)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440cdf650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e38dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ded60>, 9884132.900719928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f4fa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ed5a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a090360>, 9884135.320363244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe9a310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534596a36d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ba20210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b614d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4decf0>, 9884133.19794234)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba20850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5e120>, 9884132.731256116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440becd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadecb40>, 9884133.790453836)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e7dafd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ba06010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8ca60>, 9884132.73729166)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba06650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e73d950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9331c0>, 9884135.954987034)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f43ed10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee01f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a931e10>, 9884136.750312628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee021d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee3c850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c18d0>, 9884137.771154149)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee3c5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f5eb510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee30190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458861ef0>, 9884136.142151574)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba40710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440190210>, 9884135.438354604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec1a490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ed06f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208f000>, 9884140.551805846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ed05b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eea1990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208ef20>, 9884139.098701715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eea1e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eed55d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eeeda10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ecf90d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ed141d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c1e10>, 9884139.541511783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eed5c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208e890>, 9884138.045937756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eeed8d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a802ac0>, 9884139.026263189)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ecf9790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c24a0>, 9884138.935607567)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ed146d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343edc1910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534418297f0>, 9884140.780574275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343edc33d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ed10c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344182a0b0>, 9884141.419864321)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eea1190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee02390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8db690>, 9884142.582164291)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba54d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b8edfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9d8e50>, 9884141.871675072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b8ede50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b894c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dcde0>, 9884141.414860787)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ed45710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b9563d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193bd20>, 9884145.386513256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b956790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b966c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193b2a0>, 9884145.554185808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b967c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b592f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193a820>, 9884146.563141303)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b593e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b9ed750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b887e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b589190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b8a2150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b5a8810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b5ca090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8dae40>, 9884143.987064408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b5ca750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441939010>, 9884144.052010272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b8a2190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c7150>, 9884144.02536116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b8d4bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193a740>, 9884143.634240191)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b886f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441939da0>, 9884143.651805673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b589050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208c980>, 9884143.751662068)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b5a8cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b661a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5c0c0>, 9884154.940351536)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b661890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b4c56d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7f540>, 9884147.114910888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b4c6f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b4dbc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7de10>, 9884149.934230348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b4dbb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b8f3a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f7195c0>, 9884149.255923454)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b8f38d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b669fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b480a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b4a08d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344182a580>, 9884145.661960606)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b668350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5f690>, 9884146.105744353)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b8e0fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f932430>, 9884146.894411696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b4a06d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b3bfcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56de10>, 9884151.749790732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b3bfb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b3d64d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56c2f0>, 9884150.642257668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b3d5fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b404810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56f000>, 9884150.500077335)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b404e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b429450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56e660>, 9884151.087638084)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b3d1990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b3f2350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56f850>, 9884151.692428978)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b420e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b440e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b460d90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b4448a0>, 9884153.229697509)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba440d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b444e50>, 9884154.151467554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b39e810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b478a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b28b0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b4458d0>, 9884152.68650826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b473110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b2af050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b671710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b394b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b39f550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b488190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b59ad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56d0f0>, 9884149.931334013)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b39f410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56cb40>, 9884149.392087584)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b394a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f932f20>, 9884150.031643089)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b63e950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b4450f0>, 9884153.744598577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b281ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193ad60>, 9884149.936570266)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b5444d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56e0b0>, 9884150.07205465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b48b910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b29b390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec910d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec5f450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec78710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eafac50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef7d490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eb31910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ecdded0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eb46f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eb64150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec48510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eba3ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ebc2510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eb0e2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ecb1150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ebd5210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea20c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea49210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea5c590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea941d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eab0110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eb46c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eacbf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eacded0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eae6910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e900450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea1ca10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e91a850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e936b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e966c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e97c0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f291410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e9e8950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e83ead0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea10c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e879d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea1e250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e88c150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e8a84d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e8e7d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e721910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e73d090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e760090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e8d7910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e715e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896d400>, 9884131.139675852)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f37e450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896d550>, 9884131.183570603)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4e44d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b5af20>, 9884132.623431733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f5e9590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9f2a0>, 9884132.326798867)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f7c67d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b9ef7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ba09b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441749750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b9d0750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ba35fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ba460d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ba55fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ba5b450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ba75d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ba40690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343edf8d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee1d5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee29550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee39f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee5de50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee656d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee77ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee8b510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eea3090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee30550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee86550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eec3ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eed6fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ed28e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee66710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ed712d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ed59b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ed80210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343edaa1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ed994d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343edaef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343edfb110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ede3910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b8b03d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b8cf6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b8f2890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b915ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b91b650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b886f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b8c7ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b925b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b939b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b5ed5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b5f4490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b60ae90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b5c5ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b5c7f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b647550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b625350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b4b5590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b4da490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b50c6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b481450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b516290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b536090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b550c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b4943d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c1d30>, 9884149.935441472)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b4944d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f9314e0>, 9884152.571136566)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b2aee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441828440>, 9884149.932875255)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b8c7990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b446430>, 9884152.654264078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b88e650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed59b0>, 9884108.444114788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec90ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed5a90>, 9884109.288406922)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec5c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a37e0>, 9884108.595298233)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ecabf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c6510>, 9884111.884739265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eafab10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c41a0>, 9884110.51094768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef7e090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a3540>, 9884110.483087016)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eb32050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c7a80>, 9884109.328919457)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ecde1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed6b30>, 9884110.364225153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eb46e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed5080>, 9884111.763976512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eb64810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c6ba0>, 9884110.967178673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec48590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533faded710>, 9884110.829939771)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eba3990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadef9a0>, 9884110.884262532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ebc23d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c51d0>, 9884109.698896091)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eb0e250>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c79a0>, 9884110.706141813)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eb8f390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533faded080>, 9884110.798580268)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ebd6410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c6cf0>, 9884112.527074397)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eba2190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a801b00>, 9884112.768498912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ea498d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a800360>, 9884112.613184564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ea5c050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8032a0>, 9884113.540207984)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ea94890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8026d0>, 9884112.215553736)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eab05d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a800d00>, 9884114.578652097)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eb47e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadec210>, 9884114.577335581)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eacbe10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a802b30>, 9884114.620909085)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eacfe50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a803b60>, 9884112.869604848)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eae6fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2740>, 9884113.116134215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e9007d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadef000>, 9884114.586979536)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ea1c9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a800f30>, 9884116.105604948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e91add0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2660>, 9884115.862539256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e936a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a803cb0>, 9884115.963586802)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eb66290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f26d0>, 9884115.588125903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e97c790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df460>, 9884116.23873936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f162990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ddd30>, 9884116.706425749)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e9e8610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df310>, 9884116.77832841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e83e650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dd320>, 9884116.078058792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f033950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df4d0>, 9884117.709857915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e878610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a803d90>, 9884121.381727705)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ebd6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dc7c0>, 9884117.070460124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e88c810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a092eb0>, 9884117.624422632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e8a87d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0912b0>, 9884117.562586626)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e8e7b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a092c80>, 9884118.912242768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e721e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a091470>, 9884118.70873276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e73ce90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a091cc0>, 9884119.185742317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e760410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dcb40>, 9884117.663588434)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e8d7f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f20b0>, 9884118.358970769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e715a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d74980>, 9884134.715528728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b9ef6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4de7b0>, 9884133.558027787)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba09650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533faded390>, 9884133.406729627)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458fc8890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadecbb0>, 9884133.993568188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b1b490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a71a20>, 9884134.470019542)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba36490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2a50>, 9884135.362513471)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba06c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a0c20>, 9884134.167937104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba55c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0916a0>, 9884140.737380013)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a6a6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a093bd0>, 9884138.38136938)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba76490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2f90>, 9884134.43861799)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba3fb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0906e0>, 9884135.71053118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343edf8b90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a933e70>, 9884140.121404184)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee1da50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a930670>, 9884137.619435024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee29350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a930f30>, 9884136.301978298)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee380d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dcd00>, 9884136.841915447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee5e350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437930>, 9884137.377541663)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee64290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c0e50>, 9884136.79128701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee77dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c6dd0>, 9884137.5028091)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee8b190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c43d0>, 9884138.934449455)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eea2c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c10f0>, 9884147.145288492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee30310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c27b0>, 9884137.35588526)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee86650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c31c0>, 9884138.377267683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eec3dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cb380>, 9884143.850280497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eed71d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208f0e0>, 9884145.044928065)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ed28c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208d9b0>, 9884145.066327004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee66050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c2eb0>, 9884140.687130835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ed07ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208fd90>, 9884140.131981501)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eedb190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208d2b0>, 9884139.636496292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ed80710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c0fa0>, 9884139.866898034)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eda9fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c1f60>, 9884139.740464702)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eec1990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441829710>, 9884141.382519746)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ed83d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441828670>, 9884142.82365923)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba6fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441829160>, 9884141.676656676)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ede2090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441829240>, 9884141.40169783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b8b09d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2f20>, 9884141.930576295)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b8cf510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208d010>, 9884145.095646946)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b8f2750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9d95c0>, 9884143.464499027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b916410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9d87c0>, 9884146.029626478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b91bcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344182b700>, 9884141.9088137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b886dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208fcb0>, 9884142.367515476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b9242d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9d96a0>, 9884142.88366858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b9397d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0932a0>, 9884144.83458202)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b935d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441939630>, 9884145.455869678)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b5f6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82f150>, 9884147.095694434)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee1ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82ec10>, 9884146.430103851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b5c6190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9d9400>, 9884145.616174592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee66810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345983fee0>, 9884147.035018092)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b647190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82e7b0>, 9884148.650805123)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b627bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193a270>, 9884148.777354956)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b4b5b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7f000>, 9884148.975260906)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b4cee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f71aeb0>, 9884149.246973826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b50c510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c3770>, 9884147.773633659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b481590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534418291d0>, 9884148.246575937)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b5164d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f71b310>, 9884148.275553215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b5365d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193aa50>, 9884148.629661793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b550ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e736610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a801940>, 9884118.90078082)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e737090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b1b6090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345983c520>, 9884152.380578669)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b1a8690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b31f710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441938b40>, 9884153.66960498)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b31f7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b33d7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b446c80>, 9884154.448665287)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b1d2050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b311f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b446190>, 9884154.930087129)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b311bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b463c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed7700>, 9884154.23309778)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b462590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b2b9490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b58c610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9d710>, 9884155.167716617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b34a1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ed2e210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9dc50>, 9884154.885876823)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b561250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b322ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e70a390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345983f850>, 9884155.147842854)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b322450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eaefd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9e970>, 9884155.889047926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eaec2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea6b150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b405950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed7d20>, 9884156.482566375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ea6a610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b14e0>, 9884155.403765544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b42ba10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b443390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b301a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b310b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b3123d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b3214d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b325850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b39e710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b18b710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b1b7750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b191d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9de10>, 9884156.513009695)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b58cb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56fb60>, 9884153.593769208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b190bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82f9a0>, 9884153.542369151)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b2c2410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b445780>, 9884155.35961776)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e709010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b4470e0>, 9884154.07218756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b1b4710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b2a50>, 9884154.876756793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b440b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd692b0>, 9884151.947544236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b2f7e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc58590>, 9884152.15207876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b95ab10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534588618d0>, 9884152.937110664)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b302410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a7700>, 9884152.869788136)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b325690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c2dd0>, 9884152.95355202)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b34dcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5e580>, 9884152.0721233)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b359c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ecce310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec6dc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56fa80>, 9884157.428791044)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ecccbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0e50>, 9884159.7804706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec6c110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee69c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56c6e0>, 9884156.673105529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee69710>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343efb7210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b09f0>, 9884157.471278928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343efb7010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec16010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b2660>, 9884160.29192183)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b33ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b44a850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343efad310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030b2a0>, 9884160.0025259)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b517690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030b5b0>, 9884156.730630552)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b1c7010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef62f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c6e0>, 9884157.137996448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef604d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f0e2b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef5b410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5c360>, 9884157.455452444)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef5b390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f03f210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1f0090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5edd0>, 9884158.207322026)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b310a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5c8a0>, 9884157.527666535)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1f3a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef14a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5e6d0>, 9884158.113099707)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba2a890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f0d8750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f044e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f230>, 9884157.419644373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f0db750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5cec0>, 9884158.431518257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f0470d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef6c1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec250d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f1b00>, 9884157.572438257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e94f710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345aabab30>, 9884160.72504678)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1ba190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a803f50>, 9884155.752534462)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec24250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1cca10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5cbb0>, 9884160.63307636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eb67bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f159d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f161f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f2040>, 9884160.0077976)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f162e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f174b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df0e0>, 9884159.61631182)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f176810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f0d7090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9e430>, 9884161.245502587)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f0d6890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343edc2b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2b9610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5ecf0>, 9884161.26761336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f03d490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a39be0>, 9884162.392359463)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f2bb150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f11c110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f296410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a39400>, 9884160.372564388)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f11de90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3b620>, 9884160.628625343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f295950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f065c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4c1250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459301c50>, 9884161.546288816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f0672d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a39fd0>, 9884159.957256846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4c1690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f3f24d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a38fa0>, 9884159.978679685)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f3f0dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f3c4410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3af20>, 9884161.472091768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f3c7a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f373350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f26f410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b1c6590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f2430>, 9884160.941909784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f15a510>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3b070>, 9884160.68725544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f373510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a430>, 9884161.741250942)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f26e0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f1630>, 9884157.659562165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f2ec1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f0a6f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5de80>, 9884158.316592384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f0a5650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1d4f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a350>, 9884160.592438472)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1d4350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1ba110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5f310>, 9884162.641808564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1bc750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4ad450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f492390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a38830>, 9884161.9764052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4af410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5c600>, 9884163.690549167)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f492110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef60d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4a0d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5ce50>, 9884163.194254728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f383910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f30ad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030b310>, 9884163.189159391)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4913d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442980e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed5470>, 9884162.499493845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f437310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f158b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a740>, 9884165.428744191)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f158a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f047910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4d3d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae3d90>, 9884164.284740748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f486390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442db0c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a1440d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae04b0>, 9884163.829701684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442db27d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae2ac0>, 9884163.768236218)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a146810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f472610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae2900>, 9884163.227808118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f472550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429a5650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d767b0>, 9884164.170015756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f451b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b444360>, 9884167.751779104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4a1dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f1e10>, 9884163.839186748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f56ce90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5c910>, 9884164.33866308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429a4550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344309c510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75160>, 9884168.64459562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344309c690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f447bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5cde0>, 9884164.89038545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f447c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426ff2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d774d0>, 9884166.008173453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f2efa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344272cb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d77d90>, 9884165.295842096)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344272f150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f56c450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442791c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae2c10>, 9884169.254737625)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344241fa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442872d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a725f0>, 9884166.683599763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442873990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425f4110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae0520>, 9884165.91615132)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534425f4dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442557590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a71940>, 9884167.231899628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442556590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442501890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442986bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e79d90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429a9f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f101350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a72ba0>, 9884165.412357124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442791410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75a90>, 9884164.126052769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e6ffcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d77070>, 9884164.33753724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429a9010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a72ac0>, 9884165.901642172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343efe17d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d77620>, 9884164.407468425)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344309ccd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75b70>, 9884164.29559733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343efc70d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344231fc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a72430>, 9884168.984837703)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534424a4290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344248b290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f5e83d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314c20>, 9884168.148030072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534421ddc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5ca60>, 9884168.892521355)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f5ea690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442cbf750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f5c41d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534583147c0>, 9884168.694573408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534423ac090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314670>, 9884167.479217824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f5c6710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534428e2f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f71a430>, 9884168.152315097)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534423edb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441fa4510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d8a3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f719240>, 9884168.695772672)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441fa53d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314fa0>, 9884168.07024532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d24d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442180490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f719400>, 9884175.075561315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442183b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ddd290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ee3b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f718bb0>, 9884168.5511103)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442178990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f719ef0>, 9884168.687589465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ee1510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442203610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9faf0>, 9884169.159108102)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344248a8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534424e3090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442286d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f407bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d36c290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314360>, 9884166.822644869)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442287b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458315a90>, 9884165.88639094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534424e1e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458317e00>, 9884167.214489156)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f405f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a72c80>, 9884167.210483156)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4a0510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442554650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b9cb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5c210>, 9884170.272946173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442557710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75e80>, 9884172.021270722)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1944d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458180dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9d1d0>, 9884170.84769907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458180d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f5a3650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9dcc0>, 9884171.4626186)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458700510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441719350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896c4b0>, 9884172.900424087)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f49bad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d77690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896fc40>, 9884171.474622713)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428a47d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc68a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896d780>, 9884179.4830885)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc68710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344248b410>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5faf0>, 9884172.039068324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442489e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458800f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f071d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896f1c0>, 9884172.849743692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442312110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896eac0>, 9884172.16197834)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f06090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a6beed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896c210>, 9884171.4179816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a6bcc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344224ce10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a0b690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441bbba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534430f6110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9ec10>, 9884169.103611317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430f7e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9f620>, 9884169.624222476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441bb8ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896e9e0>, 9884171.077008598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344224ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9f930>, 9884169.642340925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f415490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f5ebd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941d630>, 9884171.28838192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b9e690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441701410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941c3d0>, 9884172.674463691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc6ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419e9810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941faf0>, 9884172.836109824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bcad590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a568d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941cd00>, 9884173.747899411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a569e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a3f9410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941c4b0>, 9884173.155680727)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a3f8390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc48750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a4b6650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896eba0>, 9884173.479176397)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc48890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a5f6b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c87c0>, 9884174.158116864)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534596c3910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441733550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a339110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f71b850>, 9884173.860942356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441733d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590ca900>, 9884173.831128577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459733510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3cdb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae3930>, 9884174.305659564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3cef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d47fd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a574390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f6a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345f65fd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534599c6d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345879a390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cbaf0>, 9884175.819081636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a4b6ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458317bd0>, 9884174.780385708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d47c410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a73a10>, 9884174.318556985)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a577850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941e350>, 9884171.949328696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f69510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941c360>, 9884171.61312908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a726e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941d780>, 9884171.590722704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a62b490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941c980>, 9884171.779117016)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458799f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459829490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9e580>, 9884175.50301528)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459829dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583dee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75be0>, 9884176.081257991)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc97010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3da990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c72a0>, 9884175.907456448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458dc1990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a70e010>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534584c5750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458203bd0>, 9884175.14886385)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a70fad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a722e0>, 9884175.697412524)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584c7950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458184b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458200590>, 9884174.741637543)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458184790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534596375d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458203c40>, 9884176.465643546)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459637190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442556b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582009f0>, 9884176.208730588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a60d390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587e63d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201780>, 9884175.55866956)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587e4b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458877810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458cc2e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201d30>, 9884175.652544504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458874e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458203e00>, 9884176.50041974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458cc1a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a3b4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c5ef0>, 9884177.448208965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a39310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d530b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a9e1490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c62e0>, 9884176.616931584)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d532f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459629650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533faded8d0>, 9884174.220133424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a9e02d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896cc90>, 9884175.84046304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d5eddd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534411a0c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441084f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c4f30>, 9884178.894340828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534411a2550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c75b0>, 9884177.10888554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534410874d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a97a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d4a1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590caeb0>, 9884177.166447308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a96dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c5160>, 9884179.210544333)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440feb190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458379490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e32d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c6510>, 9884177.364480592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a6ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6c980>, 9884177.280249305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e32710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e58c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941fc40>, 9884177.837296728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e58b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585be210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6fe70>, 9884179.374982448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585bd750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441025390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459f21850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6dc50>, 9884178.214772018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441026950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a02f0>, 9884179.201536432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459f23c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410002d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458203f50>, 9884178.192413716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440fe89d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459155d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429d5390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582000c0>, 9884178.294007095)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534590c2fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a2e40>, 9884178.506023921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d55e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441845390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a1390>, 9884177.822104454)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d671990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583c0490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c1d550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cbb60>, 9884185.12249746)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581a2b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201e80>, 9884182.280501664)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c1c4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429e7290>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c7a80>, 9884178.95574909)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e8bad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a1cb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8c130>, 9884179.764553532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a1c290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b3b550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c4910>, 9884180.277687157)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b3b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344242cad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8cde0>, 9884180.89829144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344242c410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b2ab10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9c750>, 9884179.82045276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344234bcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534417fc890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c58d0>, 9884180.173993858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534417fe4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344234aed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8fb60>, 9884180.088149112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442348810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534591be4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442335110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8d710>, 9884180.995018644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442638dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8edd0>, 9884182.650409272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442336390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427461d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6ee40>, 9884182.655011307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3d37d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441fe5010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d72580>, 9884180.473493928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441fe6590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ab9b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b95750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458be6350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d70280>, 9884181.108516358)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441aba950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582035b0>, 9884179.119358262)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b94090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9c600>, 9884178.934817052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f3ef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441949750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d71780>, 9884182.495100738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3fb650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440dc68d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d70210>, 9884186.1135774)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440dc6f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e64950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d70600>, 9884182.4715906)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e65b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344274f050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d71010>, 9884183.83139554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458875d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408dbc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e77bd0>, 9884182.750295611)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408d93d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a14050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a10f0>, 9884182.700018948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a15550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458951b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442584690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941de10>, 9884184.032224203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ec9090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442452f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c70e0>, 9884183.374259396)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442452390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344207dfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a17f0>, 9884182.542975262)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344207e510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ded9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e75be0>, 9884183.35343126)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c0d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ab22d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442230950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e777e0>, 9884183.59178369)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f30710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459511e10>, 9884183.993067775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442233210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442340dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c0f290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440af8b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8d9b0>, 9884182.235661171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442586190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d70050>, 9884181.115534976)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534418468d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534595120b0>, 9884184.777701568)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442342910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d73cb0>, 9884181.024497192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c0cd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ceef50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513e00>, 9884183.91436264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344200e4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458254910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534581b5810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459512ba0>, 9884184.242805447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441193d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513a10>, 9884186.04077352)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458211990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458af9bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ca8150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459510050>, 9884184.847149014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442397590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b46e0>, 9884188.083116325)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ca8a10>\n",
      "Evaluating workflow:  55%|█████▌    | 276/500 [00:50<00:30,  7.23it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b87990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d73c40>, 9884185.376475953)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344251c590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534424b5a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d449dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8e120>, 9884185.398525402)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534424b7690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7a80>, 9884185.33876085)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d44ac90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c66710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b44b0>, 9884185.40908302)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c66a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441763790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d72120>, 9884185.186383445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441763a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f172d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b4c20>, 9884185.253352333)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441cee550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442231550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b62e0>, 9884186.277863933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442232f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441783110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410f0510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442903890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442159210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459511860>, 9884184.398978172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442159050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b5e80>, 9884186.073347123)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441783fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458860c90>, 9884186.292212209)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534410f0590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513620>, 9884183.462188514)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442902f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f07690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534588622e0>, 9884188.421646964)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f07710>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534411d8550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ec9190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458862660>, 9884188.064067198)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a08d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d38cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d43f210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b58d0>, 9884187.743215656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d38ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458863a10>, 9884188.136805398)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d43c450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b2a490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458295150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a3c0>, 9884188.191363692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b2af10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973b230>, 9884187.849333793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458295d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534409c1b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8fa10>, 9884188.519057132)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d36bc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d6ab3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459510590>, 9884187.632887049)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d6a9990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b86d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c63b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e74bb0>, 9884187.71936176)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b87b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459739470>, 9884187.910408925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c63c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344267acd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738de0>, 9884188.205429612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442397450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f6b9f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408dbad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441005790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b6b30>, 9884187.587337097)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ec8d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458862120>, 9884186.094206356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441006010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a7b0>, 9884190.799364014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f6bb190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d720b0>, 9884185.896247879)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d547010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f7ccd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344180da90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973b380>, 9884188.794275736)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f61b050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d70d00>, 9884190.920696925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344180f390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f6383d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a1987c0>, 9884189.839272244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441902a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f65bc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a598c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19b540>, 9884195.512661876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f65b510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a2190>, 9884189.037571585)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a598710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f72b450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19a200>, 9884189.67527926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f72bd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440eb63d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459512580>, 9884190.227532608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459679ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441cef490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f8c1390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19aeb0>, 9884190.127163297)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441781410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a198d70>, 9884190.595580809)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f8c2d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f78b390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f874e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a1996a0>, 9884190.887984212)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f7881d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f837590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f07f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f71ec50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459739a20>, 9884190.966666844)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f04290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cb4d0>, 9884189.876656236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f71ead0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f77df50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f9c4b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f603510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d73b60>, 9884190.596842883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f8740d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458862f90>, 9884190.787759759)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e262d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cb230>, 9884192.95666191)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f9e6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8ec0>, 9884191.802033488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f9c4510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19a190>, 9884189.13697102)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581c8f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d352090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f939cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a1986e0>, 9884193.457177345)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f9ae650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8440>, 9884192.785521531)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f939d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f920090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8ad0>, 9884191.747086188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f921390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f730050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993c4b0>, 9884191.467246143)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f733510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f728750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993db70>, 9884194.404574247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fad9c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa8e350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993f9a0>, 9884192.29846428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa8fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa6d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993c440>, 9884194.683591831)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa6f890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f7e4450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458862e40>, 9884192.421573287)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459166e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fab1090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e771c0>, 9884193.703946676)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa04b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb97750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f045d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993f2a0>, 9884192.812222453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb97890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993ea50>, 9884194.277606124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f07d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb7dbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb62010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962f000>, 9884192.831496608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb7d850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962db00>, 9884192.840988718)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb630d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fccadd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f97fad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c80c0>, 9884191.478861948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f959c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cbb60>, 9884193.83201502)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fcc8e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc7c250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962ec10>, 9884194.511978095)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc7ce50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fbb76d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa2be90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962eeb0>, 9884194.423799315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb52c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc1ca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962e200>, 9884198.393312372)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc1d290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc31290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435860>, 9884198.368628751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc30a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fda9b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962e970>, 9884194.793440668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fdaa2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd63ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd07550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435780>, 9884196.339914396)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd627d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584360b0>, 9884195.1186696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd079d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fdab110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d5c9210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8fa0>, 9884195.865246218)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fda97d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435d30>, 9884194.66568272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d5c91d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fdc9cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ca4a0>, 9884196.417714912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fdca590>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe7a0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458436740>, 9884197.096769275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fdb1590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd92a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962df60>, 9884195.567962691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe302d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fbf28d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc9ee90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cb700>, 9884196.194911826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa2b150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993f8c0>, 9884197.024247032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd93bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962fd90>, 9884193.111401675)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fbf27d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b65f0>, 9884193.2856675)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc9d510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd2cc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff6f790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff28490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8db00>, 9884199.61363094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff6f510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e190>, 9884197.162924351)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff2b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff52190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8f380>, 9884197.82501412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff51d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa6dd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962e120>, 9884197.776559982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff6c550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440065890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585351d0>, 9884199.253463613)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440066450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344004d490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458535390>, 9884199.930304565)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400bced0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400bd090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993d5c0>, 9884200.73156644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400bc110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400d9010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534401cacd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8cfa0>, 9884200.405424643)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fdab290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973acf0>, 9884197.271220328)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd2e790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ffc69d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff85a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe16c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400d35d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7230>, 9884196.363399124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff86b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458537380>, 9884199.3688625)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534401c9710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e040>, 9884195.98890362)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ffc4fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993c9f0>, 9884196.648797808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe15750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cbee0>, 9884196.639212124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400d1f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc1c9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584355c0>, 9884198.612912076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440186e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe82ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8c7c0>, 9884199.182129094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb7ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ffc42d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b57d90>, 9884199.67333481)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344019ea10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534402a1890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa20e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585347c0>, 9884199.559333041)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534401d3190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b57a10>, 9884200.013941621)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534402bc710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff51e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b56120>, 9884199.950054027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344019f750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400f8190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440241150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c210>, 9884201.932369389)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400f8510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b56040>, 9884200.012562357)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440242650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534401fbe90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c9e80>, 9884201.275780348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534401f9c90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400ab490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584351d0>, 9884201.99310464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400a8090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440273450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585358d0>, 9884201.179072984)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344039c410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd42b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440183bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344002c150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534402b1fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344001a550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962e270>, 9884205.058127834)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534402b0550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536660>, 9884199.08499971)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344001a4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d8b40>, 9884201.38791796)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534403c7690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458535b70>, 9884199.370060192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440181090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8520>, 9884198.749252299)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344000c290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440311a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590da270>, 9884210.008041762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440310210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344012e2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993cde0>, 9884201.79520698)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534403e6e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440362f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440351250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590db4d0>, 9884203.14157012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440362e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d80c0>, 9884203.053880228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534403514d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440488bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584350f0>, 9884203.538230153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344047b850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff51b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590db310>, 9884203.887876678)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff524d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440473c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440441650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d9b00>, 9884201.794018684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440472710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962d1d0>, 9884202.353548588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440442390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400bc210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5e0b0>, 9884203.756270355)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344072b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343feb8350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440406450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5c590>, 9884202.751719106)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ffd1b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5c4b0>, 9884202.219282685)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440405790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441067490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5fcb0>, 9884203.136217969)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534410644d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534404daf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e970>, 9884205.671172244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534404d91d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c9e5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440265e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d9da0>, 9884201.698274948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534402d6e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a199ef0>, 9884203.726535343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c9e490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534404bde50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344092c690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5ef20>, 9884205.061570648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440357710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5fc40>, 9884205.981053764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344092c310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344093a450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344054bd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b547c0>, 9884205.32174011)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344093a550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1d30>, 9884204.6420492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344054ab90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534405390d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e3c0>, 9884206.592611892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440539450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bd1c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440525a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb06e0>, 9884203.94369049)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bd0890>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344039a310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1240>, 9884206.751227887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344054cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344047b8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534406d8750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1e10>, 9884207.172808444)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344047b890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f16a0>, 9884205.022797609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534406d9a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534406c0150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0910>, 9884205.798963146)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534406c0c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440aa6790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2510>, 9884206.305261554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440aa4f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344069e550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ae7410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534404bcf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d8360>, 9884205.582207104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440524d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585366d0>, 9884206.245674357)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344069e410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5d940>, 9884204.116128968)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ae4e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb3620>, 9884203.706186865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534404bcd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400f8390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440866ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f35b0>, 9884206.799490988)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407df8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2b30>, 9884207.632251209)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440603190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408a4510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407b2810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2200>, 9884207.447647916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408a4610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0a60>, 9884208.814175447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407b0510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534405cd8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2a50>, 9884207.178711792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440440b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b36d390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a79c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a190>, 9884207.453551436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b36c2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b1cbc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc2270>, 9884213.406153157)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b1cb150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a7bad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc22e0>, 9884207.814586507)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a7ba850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a7c0c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0c90>, 9884208.749187095)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a7c1050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a7d2750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1c50>, 9884209.513838142)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a7d2650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a7c3090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590dbee0>, 9884207.651039122)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a7c3510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bd9f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a828850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2dd0>, 9884208.741888111)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a7b8c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3380>, 9884210.166373849)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a828f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a7f9510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440806dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0980>, 9884209.38970886)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a79c950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3d90>, 9884206.666781705)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440805950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3770>, 9884219.980400017)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a7f9250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f2acd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e2350>, 9884208.802575871)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f2a610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a801610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a879810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3000>, 9884211.118004085)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a801510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3e70>, 9884213.220533025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a879610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a6ad5d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e32a0>, 9884210.35696078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6adf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f7599d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a6ccc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e00c0>, 9884210.307077423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6b8690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3b60>, 9884212.650871629)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6ccb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a6e3a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5e040>, 9884211.003418831)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6e3590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a6ff550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a44750>, 9884211.129282026)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6ff410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a79df90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a7416d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a44830>, 9884211.179106481)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a79e0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a475b0>, 9884211.18026392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a741d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a749250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a47380>, 9884210.905700965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a749110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a74b390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a680850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a852cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a6aaed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e0670>, 9884209.394454008)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6ab5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a457f0>, 9884212.45142104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a74b690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3230>, 9884209.700811764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a680650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a46660>, 9884210.21671282)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a852890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440467650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440373ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc56490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534596de310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d5e50d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442203e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f43fd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2d4ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f053090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b1cc0>, 9884222.39914782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b35a850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a72c7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8da20>, 9884222.286995476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440464a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993cad0>, 9884222.214615667)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440373410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8ed60>, 9884223.35261208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534597aa3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8c4b0>, 9884222.31975497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534590916d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030a5f0>, 9884222.65450487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a17550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993e3c0>, 9884222.86042378)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534425f5490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b3d20>, 9884222.621176967)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f43ed50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0280>, 9884229.685898582)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f177810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e74ad0>, 9884222.282658096)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f22fb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a478c0>, 9884211.768501792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a764850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a591450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a45630>, 9884214.930092411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a58b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b527a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b61f2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee2a4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425d0c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f380cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896cc20>, 9884222.575487817)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428c5290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ebc2810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b10090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f767ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344090a250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa17310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590daba0>, 9884222.54270639)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f7663d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d9e10>, 9884223.356959224)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534409099d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b38c0>, 9884223.21453445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b527f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896f000>, 9884222.314465668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b61f450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896ec80>, 9884222.473145)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ed96810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896d320>, 9884222.68205276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e40210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d8a60>, 9884223.297681507)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c63190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d8c20>, 9884228.170711135)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442948310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458317000>, 9884224.473187491)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa164d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a5fe090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f0076d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201470>, 9884225.9338525)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b4940d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0c20>, 9884224.378986469)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba5a810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443127010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a60d310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc1ef0>, 9884230.537084924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443011c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0210>, 9884224.471515317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a60b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a83ec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a627810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0c20>, 9884224.430790836)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a83e450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1780>, 9884225.180581203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6276d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1bfd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a6323d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962ef20>, 9884225.283124996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef3b2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b2890>, 9884229.49893108)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6321d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a640dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a477e0>, 9884225.553919785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a63b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a643450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a64bc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a473f0>, 9884224.714765033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a643310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a443d0>, 9884225.2995642)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a64bad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a6538d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb27b0>, 9884226.850674346)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6536d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e8e5910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a663cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a454e0>, 9884225.667952484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459ab5810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a45470>, 9884226.882474996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a663b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a66dd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a676410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3ee0>, 9884228.151251577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a66e0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e10f0>, 9884228.340399705)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6762d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b796290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e37e0>, 9884227.698300513)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b796850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b781f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b7a1050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c130>, 9884227.01424849)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a67b290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0750>, 9884231.55957273)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a7e9090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b791ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f1ab30>, 9884228.263496356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7b2190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a613190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f1b2a0>, 9884226.425750736)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b790750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3930>, 9884224.273418348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b9d8b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b7c7990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1e80>, 9884229.147970393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7c7850>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b7dde90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0130>, 9884238.504008967)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343edd1c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b7b1950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b834810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19860>, 9884229.662907606)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a645310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9d550>, 9884230.43255484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b82b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b802650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b58440>, 9884231.446577944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b802c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b8252d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a47d20>, 9884229.72413916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b825210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b7fb310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863d780>, 9884230.097584432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7fb190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b875a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b7b6950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b7ea850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b7fa5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a71d7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a80a910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed7e00>, 9884228.54842725)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7dd2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b59a20>, 9884230.22456456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b875690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19010>, 9884228.384110563)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7b6710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae18980>, 9884228.72167159)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7ea710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19a20>, 9884228.03883225)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7f8450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc2740>, 9884229.303832369)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7c65d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b682c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b10b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b694f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863d1d0>, 9884231.067635853)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f3dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae18de0>, 9884231.965461913)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b694d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a658610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863ce50>, 9884232.73055436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b6abfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b6c3750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed7690>, 9884234.010092476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7ec410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a653910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b6e7010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b5be70>, 9884231.37190003)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b6b2710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82edd0>, 9884232.89469908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b6e6ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b81b110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962d0f0>, 9884232.999912575)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b6dee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b6cb390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b706a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534583151d0>, 9884231.723734848)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b6cbe50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82d400>, 9884232.662895272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b706950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b715850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82c280>, 9884232.856248904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b715690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b859110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b732cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82e890>, 9884233.397441652)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b859150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82c9f0>, 9884233.705642948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b732b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b73c7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863d240>, 9884232.689119704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b73cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b750110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863edd0>, 9884233.356279766)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ebfac90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b6b74d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f931d30>, 9884232.478877136)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b750610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae182f0>, 9884231.885941263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b6b7110>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b857d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b58050>, 9884230.16375029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7d8e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b751c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f932c10>, 9884234.80789989)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b751ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b76dcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f932740>, 9884235.305663165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b76e1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a488650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a495790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f931be0>, 9884233.384422172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a488950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0360>, 9884238.86582273)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a495650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a4b01d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f1a580>, 9884234.504717221)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a4b06d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a4ac210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a4c89d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82c6e0>, 9884234.239989229)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a4ac550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d76510>, 9884239.9839336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a4c8890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a4e3b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa39d30>, 9884237.042820416)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a4e3fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a4e9b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa39160>, 9884235.5086148)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b859ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a4f2810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a50dfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa39630>, 9884237.740267348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b732950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3a190>, 9884234.98755772)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a50e710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a51b610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a528d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a482610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3b000>, 9884236.25325478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a519090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa39320>, 9884235.645996844)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a529c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f932ac0>, 9884233.739158276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a482410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a4ca510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a544650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a4f0dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a5565d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a801400>, 9884235.442712516)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b733190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3a740>, 9884235.136751294)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a544d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa38830>, 9884236.509957382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a4f0e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa380c0>, 9884236.527729373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a5563d0>\n",
      "Evaluating workflow:  56%|█████▌    | 278/500 [00:51<00:34,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 279/500 [00:51<00:33,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 281/500 [00:51<00:30,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 282/500 [00:51<00:34,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 283/500 [00:52<01:11,  3.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 284/500 [00:52<01:05,  3.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.23529411764705882, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 286/500 [00:53<00:57,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 288/500 [00:53<00:40,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 292/500 [00:54<00:33,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 293/500 [00:54<00:38,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 296/500 [00:55<00:37,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 299/500 [00:55<00:30,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 301/500 [00:55<00:25,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 304/500 [00:56<00:27,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 305/500 [00:56<00:32,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 307/500 [00:57<00:44,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.26666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 309/500 [00:57<00:33,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.11764705882352941, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 310/500 [00:57<00:40,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 311/500 [00:57<00:43,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 312/500 [00:58<00:56,  3.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 316/500 [00:59<00:39,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 317/500 [00:59<00:47,  3.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 322/500 [00:59<00:23,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 324/500 [01:00<00:21,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 327/500 [01:01<00:37,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 330/500 [01:01<00:27,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 331/500 [01:01<00:30,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 335/500 [01:02<00:23,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 339/500 [01:02<00:17,  9.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3157894736842105, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 341/500 [01:03<00:25,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 345/500 [01:03<00:22,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 346/500 [01:04<00:31,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 347/500 [01:04<00:33,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 349/500 [01:05<00:41,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 353/500 [01:05<00:21,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 355/500 [01:05<00:24,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.23529411764705882, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 360/500 [01:06<00:15,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6363636363636364, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 362/500 [01:06<00:13, 10.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 364/500 [01:06<00:14,  9.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 366/500 [01:07<00:34,  3.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 369/500 [01:08<00:26,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 372/500 [01:08<00:19,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 373/500 [01:08<00:24,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 377/500 [01:09<00:18,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.06060606060606061, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 380/500 [01:09<00:16,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 384/500 [01:10<00:15,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.11764705882352941, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 385/500 [01:10<00:15,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 386/500 [01:10<00:18,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 388/500 [01:11<00:27,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.16, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 391/500 [01:11<00:16,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 393/500 [01:12<00:17,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 396/500 [01:12<00:16,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 399/500 [01:13<00:18,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 402/500 [01:13<00:12,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 404/500 [01:13<00:10,  9.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 406/500 [01:14<00:11,  8.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 409/500 [01:14<00:14,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 411/500 [01:15<00:14,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 414/500 [01:15<00:12,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 416/500 [01:15<00:11,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 419/500 [01:16<00:13,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 420/500 [01:16<00:18,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 423/500 [01:17<00:14,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 425/500 [01:17<00:10,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 427/500 [01:17<00:14,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 429/500 [01:18<00:10,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 430/500 [01:18<00:12,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 431/500 [01:18<00:12,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 435/500 [01:18<00:07,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 439/500 [01:19<00:07,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 441/500 [01:20<00:13,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 443/500 [01:20<00:12,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 445/500 [01:21<00:10,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 446/500 [01:21<00:10,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8235294117647058, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 450/500 [01:21<00:07,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.1111111111111111, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5263157894736842, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 452/500 [01:21<00:05,  8.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 456/500 [01:22<00:05,  8.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 459/500 [01:23<00:08,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 460/500 [01:23<00:08,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 461/500 [01:24<00:08,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 467/500 [01:24<00:03,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 469/500 [01:24<00:04,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 471/500 [01:25<00:04,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 475/500 [01:25<00:02,  9.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.47619047619047616, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 477/500 [01:26<00:03,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.058823529411764705, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 479/500 [01:26<00:03,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 480/500 [01:26<00:02,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 482/500 [01:26<00:03,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 484/500 [01:27<00:03,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.375, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 486/500 [01:27<00:02,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 488/500 [01:28<00:02,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 491/500 [01:28<00:01,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2222222222222222, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 492/500 [01:28<00:01,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 493/500 [01:29<00:01,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 494/500 [01:29<00:01,  3.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 496/500 [01:30<00:01,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 497/500 [01:31<00:01,  2.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.0625, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 498/500 [01:31<00:00,  2.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 499/500 [01:33<00:00,  1.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 500/500 [01:38<00:00,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for idx in range(0,5):\n",
    "    textgrad_optimizer = TextGradOptimizer(\n",
    "        graph=workflow_graph, \n",
    "        optimize_mode=\"all\",\n",
    "        executor_llm=executor_llm, \n",
    "        optimizer_llm=optimizer_llm,\n",
    "        batch_size=3,\n",
    "        max_steps=20,\n",
    "        evaluator=evaluator,\n",
    "        eval_every_n_steps=1,\n",
    "        eval_rounds=1,\n",
    "        save_interval=None,\n",
    "        save_path=\"./\",\n",
    "        rollback=True,\n",
    "        constraints=[]\n",
    "    )\n",
    "    with suppress_logger_info():\n",
    "        results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "        out_dict = textgrad_optimizer.evaluator._evaluation_records\n",
    "        renewed_dict = {}\n",
    "        for key in out_dict.keys():\n",
    "            renewed_dict[key] = [out_dict[key]['prediction'], out_dict[key]['label'], out_dict[key]['metrics']]\n",
    "#     len(textgrad_optimizer.evaluator._evaluation_records)\n",
    "    with open(f\"./hotpot_5times_4omini_{idx}.pkl\", 'wb') as file:\n",
    "        pickle.dump(renewed_dict, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f53f86c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# textgrad_optimizer.evaluator._evaluation_records[list(textgrad_optimizer.evaluator._evaluation_records.keys())[0]]\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "1b29b5fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "d_full = []\n",
    "for idx in range(0,5):\n",
    "    d_full.append(pd.read_pickle(f\"./hotpot_5times_4omini_{idx}.pkl\"))\n",
    "out_score = []\n",
    "for prob in d_full[0].keys():\n",
    "    for iteritem in d_full:\n",
    "        if iteritem[prob][2]['acc'] == 1.0:\n",
    "            out_score.append(iteritem[prob][2])\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "44036f12",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5303038027332723"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(out_score)['f1'].sum() / 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "f8e7c3c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.438"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(out_score)['em'].sum() / 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "14276527",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.708"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(out_score)['acc'].sum() / 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "b59c472f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#generated_workflow\n",
    "qa_workflow = {\n",
    "    \"goal\": \"Provide a concise answer to the question using relevant context. The answer must be straightforward and avoid unnecessary explanations.\",\n",
    "    \"tasks\": [\n",
    "        {\n",
    "            \"name\": \"generate_answer\",\n",
    "            \"description\": \"Extract and formulate an answer from the given context.\",\n",
    "            \"inputs\": [\n",
    "                {\"name\": \"question\", \"type\": \"str\", \"required\": True, \"description\": \"The question that needs to be answered.\"},\n",
    "                {\"name\": \"context\", \"type\": \"str\", \"required\": True, \"description\": \"The background information pertinent to the question.\"}\n",
    "            ],\n",
    "            \"outputs\": [\n",
    "                {\"name\": \"answer\", \"type\": \"str\", \"required\": True, \"description\": \"The direct answer to the question.\"}\n",
    "            ],\n",
    "            \"prompt_template\": StringTemplate(instruction=\"Use the context to determine the best answer to the question. Provide your final answer in a clear format, without extra commentary or reasoning.\"),\n",
    "            \"parse_mode\": \"xml\"\n",
    "        }\n",
    "    ]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "2c62cea3",
   "metadata": {},
   "outputs": [],
   "source": [
    "api_key = \"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\"\n",
    "# Function to encode the image\n",
    "import os\n",
    "os.environ[\"OPENAI_API_KEY\"] = api_key\n",
    "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n",
    "\n",
    "\n",
    "llm_config = OpenAILLMConfig(model=\"gpt-4o-mini-2024-07-18\", openai_key=OPENAI_API_KEY, top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
    "llm = OpenAILLM(config=llm_config)\n",
    "executor_llm = OpenAILLM(config=llm_config)\n",
    "optimizer_llm = OpenAILLM(config=llm_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "d965a023",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:14.466\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.hotpotqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m51\u001b[0m - \u001b[1mloading HotPotQA data from /gpfs/radev/home/tl688/.evoagentx/data/hotpotqa/hotpot_train_v1.1.json ...\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.130\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.hotpotqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m51\u001b[0m - \u001b[1mloading HotPotQA data from /gpfs/radev/home/tl688/.evoagentx/data/hotpotqa/hotpot_dev_distractor_v1.json ...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "benchmark = HotPotQASplits()\n",
    "workflow_graph = SequentialWorkFlowGraph.from_dict(qa_workflow)\n",
    "agent_manager = AgentManager()\n",
    "agent_manager.add_agents_from_workflow(workflow_graph, executor_llm.config)\n",
    "\n",
    "evaluator = Evaluator(\n",
    "    llm=executor_llm, \n",
    "    agent_manager=agent_manager, \n",
    "    collate_func=collate_func, \n",
    "    num_workers=20, \n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "textgrad_optimizer = TextGradOptimizer(\n",
    "    graph=workflow_graph, \n",
    "    optimize_mode=\"all\",\n",
    "    executor_llm=executor_llm, \n",
    "    optimizer_llm=optimizer_llm,\n",
    "    batch_size=3,\n",
    "    max_steps=20,\n",
    "    evaluator=evaluator,\n",
    "    eval_every_n_steps=1,\n",
    "    eval_rounds=1,\n",
    "    save_interval=None,\n",
    "    save_path=\"./\",\n",
    "    rollback=True,\n",
    "    constraints=[]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "67291165",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7405"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(benchmark._fulldata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "76b45e2f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.500\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m1\u001b[0m - \u001b[1mEvaluating workflow on test set...\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.508\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae6316d5542996de7b71b87', 'answer': 'Ricard Rubio i Vives', 'question': 'Which FC Barcelona signee was a contender for the Rookie of the Year Award when he played for the Timberwolves?', 'supporting_facts': [['2011–12 Minnesota Timberwolves season', 0], ['2011–12 Minnesota Timberwolves season', 2], ['Ricky Rubio', 0], ['Ricky Rubio', 6]], 'context': [['Emilio Sagi Liñán', ['Emilio Sagi Liñán (born Bolívar, Buenos Aires, Argentina, 15 March 1900; died Barcelona, 25 May 1951), was a former Spanish footballer who played as a left-winger for FC Barcelona, the Catalan XI and Spain during the 1920s and 1930s.', \" He was the son of Emilio Sagi Barba, the Catalan baritone singer, and Concepción Liñán Pelegrí, a dancer, and as a result, was widely referred to as Sagibarba (father's surnames together in a single surname).\", ' During his playing career he played 455 games and scored 134 goals for FC Barcelona and is best remembered for forming a successful partnership with Paulino Alcántara.', ' Together with Josep Samitier, Ricardo Zamora, Félix Sesúmaga and, later, Franz Platko they were prominent members of the successful FC Barcelona team coached by Jack Greenwell.', ' His younger brother, Luís Sagi Vela, followed in his fathers footsteps and also became a successful baritone singer.', ' His son, Victor Sagi, later ran one of the biggest advertising agencies in Spain and in 1978 announced his candidacy for the presidency of FC Barcelona, but withdrew before the election was held.']], ['Ricky Rubio', ['Ricard Rubio i Vives (born October 21, 1990) is a Spanish professional basketball player for the Utah Jazz of the National Basketball Association (NBA).', ' Rubio became the youngest player ever to play in the Spanish ACB League on October 15, 2005, at age 14.', ' He made his EuroLeague debut on October 24, 2006, at age 16, becoming the first player born in the 1990s to play in a EuroLeague game.', ' He is the fifth-youngest player to make their debut in the EuroLeague.', ' On June 25, 2009, he was drafted with the fifth pick in the first round of the 2009 NBA draft by the Timberwolves, making him the first player born in the 1990s to be drafted by the NBA.', ' The Timberwolves had an agreement in principle with his former Spanish team, DKV Joventut, to buy out his contract, but Rubio backed out of the deal.', ' On August 31, 2009, Joventut traded the rights to Rubio to FC Barcelona, and Rubio signed a six-year contract with FC Barcelona the following day.', ' In 2011, Rubio joined the Minnesota Timberwolves, and spent six seasons in Minnesota before being traded to the Jazz in June 2017.']], ['Joaquim Peris de Vargas', ['Joaquim Peris de Vargas is a former President of FC Barcelona.', ' He was one of the most controversial Presidents in the history of FC Barcelona.', ' He began his career as manager in 1910 as he occupied the vice presidency, a position he held with various presidents.', ' Taking advantage of Pay Àlvar resignation in September 1914, Vargas Peris assumed leadership of FC Barcelona.', ' His spell in charge at the club was marked by constant controversy, because I always wanted to impose his opinion and even got the players rebelling against him.', ' Vargas was famous for his quote: \"I am Barcelona.\"', ' He left the organization at the request of the general assembly of FC Barcelona and he was forced to resign at the end of the season 1914-15.']], ['Ferenc Plattkó', ['Ferenc Plattkó (born Franz Platko Kopiletz in Budapest, Hungary, 2 December 1898, died Santiago, Chile, 2 September 1983), also known as Ferenc Platko or Francisco Platko, was a Hungarian footballer and manager of Austrian origin.', ' During the 1910s and 1920s he played as a goalkeeper for Vasas SC, WAC Vienna, KAFK Kula, MTK Hungária FC, FC Barcelona, Recreativo de Huelva.', ' He subsequently worked as a coach in Europe and South America, most notably with FC Barcelona, Colo-Colo, River Plate, Boca Juniors and Chile.', ' Platko was an early FC Barcelona legend and was a team-mate of Paulino Alcántara, Josep Samitier and Sagibarba.', ' His bravery as a goalkeeper was immortalized by Rafael Alberti in the poem \"Oda A Platko\".', ' After retiring as a player he returned to the club as a coach on two occasions (1934–35, 1955–56).']], ['2011–12 Minnesota Timberwolves season', ['The 2011–12 Minnesota Timberwolves season was the 23rd season of the franchise in the National Basketball Association (NBA).', ' In their first season with head coach Rick Adelman, the team finished the lockout-shortened season with a 26–40 record, nine wins above their previous season and finished in 12th place in the Western Conference.', ' This season saw the debut of 2009 draftee Ricky Rubio, who was a contender for the Rookie of the Year Award until he tore his ACL and his lateral collateral ligament after colliding into Kobe Bryant during a game against the Los Angeles Lakers and was out for the rest of the season.', ' Following the season, Brad Miller retired.']], ['Nou Palau Blaugrana', ['The Nou Palau Blaugrana will be a multi-sports indoor arena, located in Barcelona, Catalonia, Spain.', ' The arena will serve as the home arena for the basketball (FC Barcelona Bàsquet) and handball (FC Barcelona Handbol) sections of the multi-sports club FC Barcelona.', ' The Nou Palau Blaugrana will have a capacity of 12,500 spectators.']], ['Josep Maria Fusté', ['Josep Maria Fusté Blanch (born 15 April 1941) is a retired Spanish footballer and captain of FC Barcelona during the 1960s and early 1970s.', ' In 1964, together with Luis Suárez, Amancio Amaro, José Ángel Iribar and his FC Barcelona team mate, Jesús María Pereda, he also helped Spain win the European Championship.', ' He also played for CA Osasuna and Hércules CF.', ' After retiring as a player he worked as a public relations executive for \"Codorniu\", a Catalan sparkling wine company.', ' He also served as president of the FC Barcelona veterans association and publicly supported Sixto Cambra, a Catalan nationalist, who stood against Josep Lluís Nuñez in the 1989 FC Barcelona presidential elections.']], ['FC Barcelona Bàsquet B', ['FC Barcelona Bàsquet B (English: FC Barcelona Basketball B), also currently known as FC Barcelona Lassa B for sponsorship reasons, is the reserve team of FC Barcelona Lassa.', ' The team currently plays in the Spanish 2nd-tier level LEB Oro.']], ['Enrique Fernández Viola', ['Enrique Fernández Viola, commonly referred to as Enrique Fernández, (10 June 1912 – 6 October 1985) was a Uruguayan footballer and manager who played for Nacional, Talleres (RE), Independiente, FC Barcelona, Uruguay and the Catalan XI.', ' As a manager, he won two Uruguayan championships with Nacional and La Liga titles with both FC Barcelona and Real Madrid.', ' Along with Radomir Antic, he is one of only two coaches to have taken charge of both FC Barcelona and Real Madrid and he is the only coach to have won La Liga titles with both.', ' He was born in Montevideo, Uruguay.']], ['FC Barcelona Bàsquet', ['FC Barcelona Bàsquet (English: FC Barcelona Basketball), also currently known as FC Barcelona Lassa for sponsorship reasons, is a Spanish professional basketball club.', ' It is a part of the FC Barcelona multi sports club, and was founded on 24 August 1926, which makes it the oldest club in the Liga ACB.', ' The club competes domestically in the Liga ACB and the EuroLeague.', \" It has won seven of the last thirteen ACB championships, and in 2003, completed a Liga ACB (Spanish League), Copa del Rey (Spanish King's Cup) and EuroLeague triple crown.\", ' FC Barcelona Bàsquet has played in seven EuroLeague Finals, with the last one being their 2010 win.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.509\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ade4801554299728e26c6ab', 'answer': '69th Cannes Film Festival', 'question': 'Wolf and Sheep was screened at which 2016 film festival?', 'supporting_facts': [['Wolf and Sheep', 1], ['2016 Cannes Film Festival', 0]], 'context': [['2016 Cannes Film Festival', ['The 69th Cannes Film Festival was held from 11 to 22 May 2016.', ' Australian director George Miller was the President of the Jury for the main competition.', ' French actor Laurent Lafitte was the host for the opening and closing ceremonies.', ' On 15 March it was announced that Japanese director Naomi Kawase would serve as the Cinéfondation and Short Film Jury president.', ' American director Woody Allen\\'s film \"Café Society\" opened the festival.']], [\"Quiet Riot - Well Now You're Here\", [\"Quiet Riot: Well Now You're Here, There's No Way Back is a 2015 documentary film about Los Angeles-based heavy metal and hair metal band Quiet Riot.\", ' The film was directed by former actress Regina Russell Banali.', ' It premiered January 29, 2015, on Showtime and was screened out of competition at the 2015 Cannes Film Festival.', ' The film had its festival premier at the 2014 Newport Beach Film Festival and won the Festival Honors award for \"Outstanding Achievement in Filmmaking\" in the music category.', ' It also won \"Best Music Documentary\" from the Oregon Independent Film Festival, and was an official selection of The Hollywood Film Festival, the Carmel International Film Festival, Indie Memphis Film Festival, St. Louis International Film Festival, and Big Sky Documentary Film Festival.']], ['Loopy (film)', ['Loopy is a 2004 film written and directed by Seth Michael Donsky.', ' It is an adaptation of a short story by Ruth Rendell.', ' \"Loopy\" screened at the Palm Springs International Festival of Short Films, the Cinequest Film Festival and the Clermont-Ferrand International Short Film Festival.', ' \"Loopy\" currently airs in rotation on the Independent Film Channel.', ' The tagline for the film is \"A sheep in wolf\\'s clothing!\"']], ['Dominique Cabrera', ['Dominique Cabrera (born 21 December 1957) is a French film director.', ' She has taught filmmaking at La Fémis and at Harvard University.', ' Her film \"Nadia et les hippopotames\" was screened in the Un Certain Regard section at the 1999 Cannes Film Festival.', ' Additionally, her work has screened in Berlin International Film Festival, the Toronto International Film Festival, the Vienna International Film Festival, the Locarno Film Festival, the Rotterdam Film Festival, and in the New York Film Festival, among others.']], ['The Herd (1978 film)', ['The Herd (Turkish: \"Sürü\" ) is a 1978 Turkish drama film, written, produced and co-directed by Yılmaz Güney with Zeki Ökten during Güney\\'s second imprisonment, featuring Tarık Akan as a peasant, forced by a local blood feud to sell his sheep in far away Ankara.', ' The film, which went on nationwide general release on \\xa027,\\xa01978\\xa0(1978--) , was screened in competition at the 30th Berlin International Film Festival, where it won Interfilm and OCIC Awards, the Locarno International Film Festival, where it won Golden Leopard and Special Mention, was scheduled to compete in the cancelled 17th Antalya Golden Orange Film Festival, for which it received 6 Belated Golden Oranges, including Best Film and Best Director, was awarded the BFI Sutherland Trophy and was voted one of the 10 Best Turkish Films by the Ankara Cinema Association.']], ['Sheep and Wolves', ['Volki i ovtsy.', ' Be-e-e-zumnoe prevrashchenie (Russian: Волки и овцы: бе-е-е-зумное превращение ), released in English-speaking territories as Sheep & Wolves, is a 2016 Russian computer-animated fantasy-comedy film with a story containing elements of the fairytale \"The Wolf and the Seven Young Kids\" and the concepts of a Wolf in sheep\\'s clothing.', ' It was directed by Andrey Galat and Maxim Volkov, and stars Alexander Petrov as Grey, a young careless wolf who is wanted by the others in a pack to become its next leader.', ' However, he turns in a ram after drinking a potion he received from a rabbit named Mami (Tatyana Shitova), which leads him to have to protect a colony of sheep from the wolf pack.']], ['Maya Gallus', ['Maya Gallus is a Canadian documentary filmmaker, and co-founder of Red Queen Productions with Justine Pimlott.', ' Her films have screened at international film festivals, including Toronto International Film Festival, Montreal World Film Festival, Hot Docs Canadian International Documentary Festival, Sheffield Doc/Fest, SEOUL International Women’s Film Festival, Singapore International Film Festival, This Human World Film Festival (Vienna) and Women Make Waves (Taiwan), among others.', ' Her work has also screened at the Museum of Fine Arts (Boston), Donostia Kultura, San Sebastián and Canada House UK, as well as theatrically in Tokyo, San Francisco, Key West and Toronto, and been broadcast around the world.', ' She has won numerous awards, including a Gemini Award for Best Direction for Girl Inside, and has been featured in The Guardian, UK; Ms. (Magazine), Curve (Magazine), Bust (Magazine), Salon (Magazine), POV (Magazine) and The Walrus, among others.', ' She is a Director/Writer alumna of the Canadian Film Centre and a participant in Women in the Director’s Chair.', ' She will be honoured with a \"Focus On\" retrospective at the 2017 Hot Docs festival.']], ['Leonardo Corbucci', ['Leonardo Corbucci is an award-winning film director from Italy who is now located in Los Angeles US and member of the DGA (Directors Guild of America).', ' He has won many prizes from the prestigious \"Rising Star\" at the Canadian International Film Festival to the \"Silver Ace\" at the Las Vegas International Film festival, \"Best Romance\" at the Best of Best Film Fest, \"Best Experimental Film\" at the Idie Gathering, \"Best Drama\" at the International Family Film Festival, \"Best screenplay\" at the Sunset Film Festival, \"Honorable Mention\" at the Queen World Film Festival, \"Titoli Price\" at the Murgia Film Festival, \"Best Short film\" at the Burbank Film Festival.', ' His films have been official selections at the Venice Film Festival, Beverly Hills Film Festival, Santa Barbara Film Festival, San Diego IndieFest, Phoenix Film Festival, Riverside Film Festival, Seattle True Independent Film Festival, Ventura Film Festival, Rome Film Festival, Riverband Film Festival and screened at that Cannes Film Festival.']], ['The Immortals (2015 film)', ['The Immortals directed by Shivendra Singh Dungarpur premiered at the 20th Busan International Film Festival in October 2015 and was also shown at the 17th Jio MAMI Mumbai Film Festival.', ' It was screened as the opening film in the Documentary Section at the 21st Kolkata International Film Festival.', ' \"The Immortals\" (2015) was selected for the National Competition Section at the 14th Mumbai International Film Festival (MIFF) 2016 and won the Special Jury Award that was presented to Shivendra Singh Dungarpur at the closing ceremony on February 3, 2016.', ' The Immortals will be screened at the 30th edition of the Il Cinema Ritrovato Festival in Bologna in June 2016.']], ['Wolf and Sheep', ['Wolf and Sheep is a 2016 Danish-Afghan drama film directed by Shahrbanoo Sadat.', \" It was screened in the Directors' Fortnight section at the 2016 Cannes Film Festival where it won the Art Cinema Award.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.509\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac097b05542996f0d89cc18', 'answer': 'yes', 'question': 'Are Ruggero Deodato from Italy, and Mexican Alejandro Springall, both film directors?', 'supporting_facts': [['Ruggero Deodato', 0], ['Alejandro Springall', 0]], 'context': [['John Steiner', ['John Steiner (born 7 January 1941 in Chester, Cheshire) is an English actor.', ' Tall, thin and gaunt, Steiner attended the Royal Academy of Dramatic Arts and worked for a few years at the BBC.', ' Steiner featured in a lead role in a television production of \"Design for Living\" by Noël Coward.', ' Later he found further work primarily in films including \"Marat/Sade\" (1967), and the original \"Bedazzled\" (1967) with Peter Cook and Dudley Moore.', ' In 1969, Steiner was hired to play a part in the spaghetti western \"Tepepa\", and also appeared opposite Franco Nero in \"White Fang\", directed by Lucio Fulci.', ' He found himself in demand in Italy and moved there, appearing in a great number of Italian exploitation and B-films including police actioners (\"Violent Rome\"), westerns (\"Mannaja\"), war films (\"The Last Hunter\"), nazisploitation (\"Deported Women of the SS Special Section\"), sci-fi adventure films (\"Yor, the Hunter from the Future\"), and horror films, such as Mario Bava\\'s \"Shock\", Dario Argento\\'s \"Tenebrae\", and Ruggero Deodato\\'s \"Body Count\".', ' He also became a favourite of famed Italian filmmaker Tinto Brass, featuring in \"Salon Kitty\", the infamous \"Caligula\", \"Action\", and \"Paprika\".']], ['Hercules, Prisoner of Evil', ['Hercules, Prisoner of Evil (Italian: \"Ursus il terrore dei Kirghisi\" , \\'Ursus, Terror of the Kirghiz\\' ) is a 1964 Italian \"peplum\" film directed by Antonio Margheriti and an uncredited Ruggero Deodato.', ' Deodato, the official assistant director, replaced Margheriti as he was busy with the completion of the film \"The Fall of Rome\".', ' Deodato actually directed most of the film in actuality but Margheriti was credited as the director.', ' The film is filled with a variety of horrific themes and elements, featuring a killer werewolf, and is as much a horror film as it is a peplum.']], ['Ruggero Deodato', ['Ruggero Deodato (born 7 May 1939) is an Italian film director, screenwriter, and actor.']], ['Body Count (1987 film)', ['Body Count (released in Italy as Camping del Terrore/ Camping Terror) is a 1986 slasher film directed by Ruggero Deodato.', ' It was released in Germany as \"Body Count: Die Mathematik des Schreckens\", and in Denmark as \"Shamen\".']], ['Alejandro Springall', ['Alejandro Springall is a Mexican film director and producer.']], ['Cannibal Holocaust', ['Cannibal Holocaust is a 1980 Italian cannibal horror film directed by Ruggero Deodato from a screenplay by Gianfranco Clerici.', ' It stars Carl Gabriel Yorke, Robert Kerman, Francesca Ciardi, Perry Pirkanen, and Luca Barbareschi.', ' Influenced by the works of Mondo director Gualtiero Jacopetti, the film was inspired by Italian media reporting of Red Brigade terrorism.', \" The coverage included news reports Deodato believed to be staged, an idea which became an integral aspect of the film's story.\", ' \"Cannibal Holocaust\" was filmed primarily in the Amazon rainforest of Colombia with indigenous tribes interacting with American and Italian actors.']], ['Cannibal boom', ['The cannibal boom is a period in the history of exploitation film, lasting roughly from 1977 to 1981, where cannibal films were at the peak of their popularity in Grindhouse theaters and cinema.', ' Though Umberto Lenzi started the cannibal genre with his film \"Man from Deep River\" in 1972, it was not until Ruggero Deodato released his film \"Last Cannibal World\" in 1977 that the concept of cannibal films began to catch on.', ' Although five cannibal films were made in 1977 and 1978, none were released in 1979 (though Deodato\\'s \"Cannibal Holocaust\" was in the works).', ' In February 1980, Deodato released \"Cannibal Holocaust\", which was the start of a chain of seven similar films to be made and released in the same year.', ' The following year, however, in 1981, only two cannibal films were made (one of them was \"Cannibal Ferox\", second in notoriety only to \"Cannibal Holocaust\").', ' Only four other cannibal films were made after 1981 until the fad\\'s conclusion in 1988 with Antonio Climati\\'s \"Natura Contro\".']], ['No eres tú, soy yo', ['No eres tú, soy yo (English: It\\'s not you, it\\'s me ) is a 2010 Mexican romantic comedy directed by Alejandro Springall starring Eugenio Derbez, Alejandra Barros and Martina García and based on the Argentinean film \"No sos vos, soy yo\" (2004), written by Juan Taratuto.', ' It was produced by Matthias Eherenberg and filmed in Mexico City.']], ['Annie Belle', ['Annie Brilland (born 10 December 1956) is a French actress and social worker.', \" Her acting career began in 1974 and throughout the seventies, has had a series of varied roles in both French and Italian cinema, working with such directors as Jean Rollin, Ruggero Deodato and Joe D'Amato.\"]], [\"Concorde Affaire '79\", ['Concorde Affaire \\'79 (1979) (Italian: \"Affare Concorde\" ), also known as The Concorde Affair, is an Italian action thriller directed by Ruggero Deodato and written by Ernesto Gastaldi and Renzo Genta.', ' Released in the same year as \"The Concorde ... Airport \\'79\" and featuring actor Joseph Cotten, who appeared in \"Airport \\'77\", the film was an attempt by producers to take advantage of the success of the \"Airport\" film franchise of the 1970s.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.509\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a9064c755429916514e74a6', 'answer': 'Lester', 'question': \"What is the first name of Jack Benny Binion's father?\", 'supporting_facts': [['Jack Binion', 0], ['Jack Binion', 1], ['Benny Binion', 0]], 'context': [['Milt Josefsberg', ['Milt Josefsberg (June 29, 1911 – December 14, 1987) was a radio writer for \"The Jack Benny Program\", and later a writer for many television sitcoms, such as \"Archie Bunker\\'s Place\", \"All in the Family\", \"Here\\'s Lucy\", \"The Lucy Show\", \"The Odd Couple\" and \"The Jack Benny Show\".', ' He wrote books on the Jack Benny Show and comedy writing.', ' Jim Wickey of \"The Green House\", \"The Rip Borsley Show\" once commented about Josefsberg:']], ['The Mouse That Jack Built', ['The Mouse That Jack Built is a 1959 Warner Bros. \"Merrie Melodie\" cartoon short starring Jack Benny and the regular cast of \"The Jack Benny Program\" as mice.', ' It was written by Tedd Pierce and directed by Robert McKimson, with music by Milt Franklyn.']], ['World Poker Tour bracelet', [\"The World Poker Tour bracelet is the World Poker Tour's (WPT) answer to the World Series of Poker (WSOP) bracelet.\", ' Beginning in 1976, the WSOP started awarding bracelets to winners of WSOP events.', ' The WSOP bracelet has become synonymous with greatness.', ' \"It\\'s impossible to overstate the value of a World Series of Poker gold bracelet to anyone who takes the game seriously,\" stated World Series of Poker Commissioner Jeffrey Pollack during the 2006 bracelet unveiling.', ' \"It is the equivalent of winning the Stanley Cup in hockey or the Lombardi Trophy in football.\"', \" Since their introduction, a poker player's success has been measured by the number of bracelets they had won.\", ' With introduction of the WPT bracelet, the WPT hopes to capitalize on the prestige of winning poker bracelets.', ' WPT Founder, President and CEO, Steve Lipscomb said, \"The championship bracelet has become synonymous with poker as a symbol of achievement and respect, and we are honored to continue the tradition that Benny Binion [the founder of the WSOP] began over 30 years ago.\"']], ['Jack Binion', ['Jack Benny Binion (born February 21, 1937 in Dallas, Texas) is an American businessman.', \" Binion is the son of casino magnate Benny Binion and worked for his father at Binion's Horseshoe, a casino and hotel in Las Vegas, Nevada.\"]], ['Horseshoe Casino Tunica', ['The Horseshoe Casino Tunica is a casino resort located in Tunica Resorts, Mississippi.', \" It was developed by Jack Binion, the son of Las Vegas gaming legend Benny Binion and named after his father's famous Binion's Horseshoe downtown gambling hall.\", ' Much like its namesake, the Horseshoe Tunica is known for catering to serious gamblers, particularly table games players, and is known for its liberal, player-favorable rules and its comp policies.']], ['Jack Benny filmography', ['This is a list of movies featuring comedian Jack Benny.', \" Benny's career lasted from the early 1900s until his death in 1974.\", ' In Jack Benny\\'s first film he starred along with Conrad Nagel as master of ceremonies in \"The Hollywood Revue of 1929\", which was a big role for Jack at the time.', \" Benny wouldn't start getting well known until his own radio program in 1934.\", ' \"The Hollywood Revue\" is also the oldest known form of Jack Benny in color with the last sequence being filmed originally in color, which was common for a musical in 1929.']], [\"Binion's Gambling Hall and Hotel\", [\"Binion's Gambling Hall & Hotel, formerly Binion's Horseshoe, is a casino on the Fremont Street Experience in Downtown Las Vegas, Nevada.\", ' It is owned by TLC Casino Enterprises.', ' The casino is named for its founder, Benny Binion, whose family ran it from its founding in 1951 until 2004.', ' The hotel, which had 366 rooms, closed in 2009.']], ['Benny Binion', ['Lester Ben \"Benny\" Binion (November 20, 1904 – December 25, 1989) was an American gambling icon and mob boss.']], ['U.S.A. Confidential', ['U.S.A. Confidential is a 1952 book written by Jack Lait and Lee Mortimer (Crown Publishers).', ' Its theme is crime and corruption.', ' The book is remarkable for early mentions of many who would become infamous, among them Benny Binion and Jimmy Fratianno.']], ['Poker Hall of Fame', ['The Poker Hall of Fame is the hall of fame of professional poker in the United States.', \" Founded in Las Vegas, it was created in 1979 by Benny Binion, the owner of the Horseshoe Casino, to preserve the names and legacies of the world's greatest poker players and to serve as a tourist attraction to his casino.\", ' Binion was known for the creative ways in which he marketed his casino.', ' In 1949, he convinced Johnny Moss and Nick \"The Greek\" Dandolos to play high-stakes poker heads up where the public could watch them.', ' In 1970, he invited a group of poker players to compete in what would be the first World Series of Poker (WSOP).', \" When Harrah's Entertainment, now known as Caesars Entertainment, acquired the rights to the WSOP in 2004, it also assumed ownership of the Poker Hall of Fame.\", ' Currently, membership in the Poker Hall of Fame is handled directly by the WSOP.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.510\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a79caad5542994f819ef09f', 'answer': 'Matthew Ryan Kemp', 'question': 'Which professional baseball player was born in 1984 and played as a rookie for the Los Angeles Dogers in 2007?', 'supporting_facts': [['2007 Los Angeles Dodgers season', 0], ['2007 Los Angeles Dodgers season', 3], ['Matt Kemp', 0]], 'context': [['Steve Garvey', ['Steven Patrick Garvey (born December 22, 1948) is an American former professional baseball player and current Southern California businessman.', ' He played in Major League Baseball as a first baseman, most notably for the Los Angeles Dodgers.', ' Nicknamed \"Mr. Clean\" because of the wholesome image he portrayed throughout his career in baseball, Garvey was the 1974 National League Most Valuable Player Award winner, a two-time National League Championship Series MVP (1978 and 1984), a 10-time All-Star, and a two-time MVP of the All-Star Game (1974 and 1978).', ' He holds the National League record for consecutive games played (1,207).']], ['Matt Kemp', ['Matthew Ryan Kemp (born September 23, 1984) is an American professional baseball outfielder for the Atlanta Braves of Major League Baseball (MLB).', ' He began his professional career in the Los Angeles Dodgers organization in 2003, and played with the Dodgers from 2006 until 2014.', ' He played for the San Diego Padres in 2015 and 2016.', ' He has been named to two All-Star teams and has won two Gold Glove Awards (2009 and 2011) and two Silver Slugger Awards (2009 and 2011).']], ['Ernie Orsatti', ['Ernest Ralph Orsatti (September 8, 1902 in Los Angeles, California – September 4, 1968 in Canoga Park, California), was a professional baseball player who played outfielder and first base in the Major Leagues from 1927 -1935 .', ' He played for the St. Louis Cardinals with a lifetime batting average of .306 in 701 games.', ' He both batted and threw left-handed.', ' He played in the 1928, 1930, 1931 and 1934 World Series.', ' He previously played for the 1925 Cedar Rapids, Iowa Bunnies minor league team with a batting average of .347 and 6 home runs.', ' He attended Los Angeles Manual Arts High School.', ' He joined his brothers in the Orsatti Talent Agency after retiring from baseball.']], ['Joel Guzmán', ['Irvin Joel Vigo Guzmán (born November 24, 1984) is a Dominican professional baseball player.', ' He currently plays for the York Revolution of the Atlantic League.', ' He has played in Major League Baseball (MLB) for the Los Angeles Dodgers and Tampa Bay Devil Rays and in Nippon Professional Baseball (NPB) for the Chunichi Dragons.']], ['Mike Marshall (outfielder)', ['Michael Allen Marshall (born January 12, 1960) is an American former professional baseball player and current commissioner of the Pacific Association of Professional Baseball Clubs.', ' He played as an outfielder in Major League Baseball for the Los Angeles Dodgers, New York Mets, Boston Red Sox and California Angels from 1981 to 1991.', ' He also played one season in Nippon Professional Baseball for the Nippon Ham Fighters in 1992.', ' Marshall served as president and general manager of the Chico Outlaws of the North American League.']], ['Pete Rose Jr.', ['Peter Edward \"PJ\" Rose Jr. (born November 16, 1969) is the manager of the Wichita Wingnuts in the American Association of Independent Professional Baseball and former professional baseball player.', \" The son of Major League Baseball's all-time hits leader Pete Rose, Rose Jr. played in the minor leagues most of his career except for a brief stint in for the Cincinnati Reds.\", ' He was released September 14, 2009, by the York Revolution of the Atlantic League of Professional Baseball.', ' In 2011, he joined the White Sox coaching staff and became the manager of their Appalachian League (rookie league) affiliate in Bristol.', ' In 2012, he moved up to the Pioneer League with the affiliate in Great Falls, Montana.', ' After one season, he advanced to the lower-A South Atlantic League team in Kannapolis, North Carolina.']], ['Billy Bean', ['William Daro Bean (born May 11, 1964) is an American former professional baseball player.', ' He played in Major League Baseball (MLB) as an outfielder for the Detroit Tigers (1987–1989), Los Angeles Dodgers (1989), and San Diego Padres (1993–1995), as well as the Kintetsu Buffaloes of Nippon Professional Baseball in 1992.']], ['2007 Los Angeles Dodgers season', [\"The Los Angeles Dodgers' 2007 season started off promisingly with the Dodgers holding the Western Division lead for most of the first half of the season.\", ' However, the team faded down the stretch and finished the season in fourth place.', ' Two of the teams big free agent signings, pitchers Jason Schmidt and Randy Wolf were injured and missed most of the season.', ' A promising development was the play of several rookies including James Loney and Matt Kemp and the further development of second year catcher Russell Martin, who was named to his first All-Star Game.']], ['Bob Bailey (baseball)', ['Robert Sherwood Bailey (born October 13, 1942) is an American former professional baseball third baseman.', ' He played seventeen seasons in Major League Baseball (MLB) between 1962 and 1978 for the Pittsburgh Pirates, Los Angeles Dodgers, Montreal Expos, Cincinnati Reds, and Boston Red Sox.', ' Bailey attended Wilson Classical High School, where he was the 1961 CIF Baseball Player of the Year and quarterbacked the football team for two years, one of which was undefeated.', ' He was originally signed by the Pittsburgh Pirates as a bonus baby.', ' After the 1966 season, the Pirates traded Bailey and Gene Michael to the Los Angeles Dodgers for Maury Wills.']], ['Shoki Kasahara', [\"Shoki Kasahara (笠原 将生 , Kasahara Shōki ) (born January 9, 1991 in Fukuoka) is a Japanese professional baseball player who last played for the Yomiuri Giants in Japan's Nippon Professional Baseball.\", ' His younger brother Taiga is also a professional baseball player currently playing for Fukuoka SoftBank Hawks.', ' His father Eiichi is a former professional baseball player.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.510\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5addc79d5542995b365fab7b', 'answer': 'To SquarePants or Not to SquarePants', 'question': 'Which episode of SpongeBob SquarePants aired first, The Clash of Triton or To SquarePants or Not to SquarePants?\"', 'supporting_facts': [['The Clash of Triton', 0], ['The Clash of Triton', 1], ['SpongeBob SquarePants (season 6)', 5]], 'context': [['The SpongeBob SquarePants Movie', ['The SpongeBob SquarePants Movie is a 2004 American live-action/animated comedy film based on the Nickelodeon television series \"SpongeBob SquarePants\".', \" The film was co-written, directed, and co-produced by series creator Stephen Hillenburg and starred the series' cast of Tom Kenny, Bill Fagerbakke, Clancy Brown, Rodger Bumpass and Mr. Lawrence, with guest performances by Scarlett Johansson, Jeffrey Tambor, Alec Baldwin and David Hasselhoff.\", ' It was produced by Hillenburg\\'s production company United Plankton Pictures and Nickelodeon Movies, it was distributed by Paramount Pictures and was also the first film in the \"SpongeBob SquarePants\" film series.', \" In the film, Plankton devises a plan to steal King Neptune's crown and send it to Shell City, and SpongeBob and Patrick must retrieve the crown to save Mr. Krabs from King Neptune's wrath and Bikini Bottom from Plankton's plan.\"]], ['The Clash of Triton', ['\"The Clash of Triton\", also known as \"Neptune\\'s Party\", is the 26th episode of the sixth season and the 126th overall episode of the American animated television series \"SpongeBob SquarePants\".', ' It originally aired on Nickelodeon in the United States on July 5, 2010.']], [\"SpongeBob SquarePants: Plankton's Robotic Revenge\", ['SpongeBob SquarePants: Plankton\\'s Robotic Revenge is an action-adventure video game based on the television series \"SpongeBob SquarePants\".', ' It was released in October 2013 for Wii U, Wii, Nintendo 3DS, Nintendo DS, PlayStation 3, and Xbox 360.', ' The game was developed by Behaviour Interactive and published by Activision, who took over the license from previous \"SpongeBob SquarePants\" video game publisher THQ after the company\\'s bankruptcy and liquidation.']], ['List of SpongeBob SquarePants guest stars', ['In addition to the show\\'s regular cast of voice actors, guest stars have been featured on \"SpongeBob SquarePants\", an American animated television series created by marine biologist and animator Stephen Hillenburg for Nickelodeon.', ' \"SpongeBob SquarePants\" chronicles the adventures and endeavors of the title character and his various friends in the fictional underwater city of Bikini Bottom.', ' Many of the ideas for the show originated in an unpublished, educational comic book titled \"The Intertidal Zone\", which Hillenburg created in the mid-1980s.', ' He began developing \"SpongeBob SquarePants\" into a television series in 1996 upon the cancellation of \"Rocko\\'s Modern Life\", which Hillenburg directed.', ' The pilot episode first aired on Nickelodeon in the United States on May 1, 1999.', ' The show\\'s ninth season premiered in 2012, and episodes of \"SpongeBob SquarePants\" have aired.', ' A feature-length film adaptation of the show, \"The SpongeBob SquarePants Movie\", was released in 2004; in 2015, a sequel, \"\", was released.']], ['SpongeBob SquarePants: Lights, Camera, Pants!', ['SpongeBob SquarePants: Lights, Camera, Pants!', ' is a 2005 party video game based on the TV series \"SpongeBob SquarePants\".', ' It was released in October 2005 for the Xbox, PlayStation 2, GameCube, Game Boy Advance, and the PC.', ' It was released for the Nintendo DS in Korea in 2007, but its North American release was cancelled.', ' It is the first \"SpongeBob SquarePants\" title to feature multiplayer mini-games, similar to the \"Mario Party\" video game series.', ' It is also the last SpongeBob game for the Xbox.', ' It is also the last time Charles Nelson Reilly would voice the Dirty Bubble before his death in 2007.', ' For reasons unknown, Mermaid Man was not voiced by his original voice actor Ernest Borgnine but instead Joe Alaskey, who would voice him again in .']], ['SpongeBob SquarePants 4D: The Great Jelly Rescue', ['SpongeBob SquarePants 4D: The Great Jelly Rescue (often referred to as SpongeBob SquarePants 4D or simply The Great Jelly Rescue) is a 4-D film attraction that serves a sequel to SpongeBob SquarePants 4-D.', ' It follows SpongeBob, Patrick, and Sandy as they go jellyfishing.']], ['SpongeBob SquarePants (season 6)', ['The sixth season of the American animated television series \"SpongeBob SquarePants\", created by former marine biologist and animator Stephen Hillenburg, aired on Nickelodeon from March 3, 2008 to July 5, 2010, and contained 26 episodes, beginning with the episode \"Krabby Road\".', ' The series chronicles the exploits and adventures of the title character and his various friends in the fictional underwater city of Bikini Bottom.', ' The season was executive produced by series creator Hillenburg and supervising producer Paul Tibbitt, who also acted as the showrunner.', ' In 2009, the show celebrated its tenth anniversary on television.', ' The documentary film titled \"\" premiered on July 17, 2009, and marked the anniversary. \"', 'SpongeBob\\'s Truth or Square\", a television film, and the special episode \"To SquarePants or Not to SquarePants\" were broadcast on Nickelodeon, as part of the celebration.']], ['The SpongeBob Movie: Sponge Out of Water', ['The SpongeBob Movie: Sponge Out of Water is a 2015 American 3D live-action/animated comedy film based on the animated television series \"SpongeBob SquarePants\".', ' A stand-alone sequel to \"The SpongeBob SquarePants Movie\" (2004), it was directed by former series showrunner Paul Tibbitt in his directorial debut, with live-action sequences directed by Mike Mitchell.', ' It was the first film to be produced by Paramount Animation and second film in the \"SpongeBob SquarePants\" film series.', \" The film stars Antonio Banderas and features the show's regular voice cast, who returned to reprise their respective roles from the series and the previous film.\", ' The plot follows a pirate called Burger-Beard, who steals the Krabby Patty secret formula using a magical book that makes any text written upon it come true.', ' SpongeBob and his friends must travel to the surface to confront Burger-Beard and get the formula back.']], ['SpongeBob SquarePants 4-D', ['SpongeBob SquarePants 4-D (also known as SpongeBob SquarePants 4-D Ride, SpongeBob SquarePants: The Ride or SpongeBob SquarePants 3-D) is a cel-shaded 4-D film based upon the popular television series \"SpongeBob SquarePants\".', ' It can be found at many aquariums and theme parks across the world.', ' The ride consists of a pre-show which then leads into a stadium seated auditorium.', ' The ride is in 4-D, meaning it is a motion simulator with a 3D movie.', ' The effects on the ride vary at different parks.', ' Water spray, bubbles, wind, leg ticklers, smoke, and smells are usually found.']], ['SpongeBob SquarePants: Original Theme Highlights', ['SpongeBob SquarePants: Original Theme Highlights is the first album of songs played on the Nickelodeon TV series \"SpongeBob SquarePants\".', \" It includes tracks sung by the cartoon's characters: SpongeBob SquarePants, Sandy Cheeks, Patrick Star, Squidward Tentacles, and Plankton.\", ' Its total running time is 9 minutes and 9 seconds, spanning seven tracks.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.511\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a83a31c554299123d8c2179', 'answer': 'Adam Levine', 'question': \"Who is the lead vocalist for Maroon 5's sixth studio album?\", 'supporting_facts': [['What Lovers Do', 1], ['Maroon 5', 1]], 'context': [[\"Don't Wanna Know\", ['\"Don\\'t Wanna Know\" is a song by American pop rock band Maroon 5.', ' It features guest vocals from American rapper Kendrick Lamar.', \" The song was released on October 11, 2016, as the lead single from the band's upcoming eponymous sixth studio album Maroon 5 (2017).\", ' The song reached the top ten in 15 countries, including number six in the United States.']], [\"Heard 'Em Say\", ['\"Heard \\'Em Say\" is a song by American hip-hop artist Kanye West.', ' It was released on November 8, 2005 as the third single for his second studio album, \"Late Registration\".', ' The song features Maroon 5 lead singer Adam Levine, who sings the chorus and bridge.', ' West co-produced the track with Jon Brion, and it contains excerpts from \"Someone That I Used to Love\" as performed by Natalie Cole.', \" The song was met by acclaim from contemporary music critics and is often seen as one of West's best.\", ' The refrain of \"Heard \\'Em Say\" was later recycled for the Maroon 5 song \"Nothing Lasts Forever\" which appears on their 2007 album \"It Won\\'t Be Soon Before Long\".']], ['If I Never See Your Face Again', ['\"If I Never See Your Face Again\" is a song by American pop rock band Maroon 5 from the June 2008 re-release of the group\\'s second studio album, \"It Won\\'t Be Soon Before Long\" (2007).', ' It was also included on the June 2008 \"Good Girl Gone Bad\" (2007).', ' The song was originally included on the standard version of the album without the inclusion of Rihanna.', ' It was written by band members Adam Levine and James Valentine, with production of the song helmed by Christopher \"Tricky\" Stewart, Mike Elizondo, Mark Endert, Mark \"Spike\" Stent and Maroon 5.', ' It was released as an Extended Play (EP) in Australia on May 22, 2007, and as an official single on May 2, 2008, in the United States.']], ['Maroon 5', ['Maroon 5 is an American pop rock band that originated in Los Angeles, California.', ' It currently consists of lead vocalist Adam Levine, keyboardist and rhythm guitarist Jesse Carmichael, bassist Mickey Madden, lead guitarist James Valentine, drummer Matt Flynn and keyboardist PJ Morton.']], ['What Lovers Do', ['\"What Lovers Do\" is a song by American pop rock band Maroon 5 featuring American R&B singer Sza.', \" It was released on August 30, 2017, as the third single from the band's upcoming sixth studio album (2017).\", ' The song contains an interpolation of the 2016 song \"Sexual\" by Neiked featuring Dyo, therefore Victor Rådström, Dyo and Elina Stridh are credited as songwriters.']], [\"Maroon 5's sixth studio album\", ['Maroon 5 is the forthcoming eponymous sixth studio album by American pop rock band Maroon 5, scheduled for release on November 3, 2017 by 222 and Interscope Records.']], ['Feelings (Maroon 5 song)', ['\"Feelings\" is a song recorded by American pop rock band Maroon 5 for their fifth studio album, \"V\" (2014).', ' It was written by Adam Levine, Shellback, and Oscar Göres and produced by the latter two.', ' It was sent to U.S. Adult Contemporary and contemporary hit radio on September 14 and 15, respectively, as the fourth single from the album.', \" The official artwork for the single was unveiled by Maroon 5's official Twitter account on October 6, 2015.\", ' Although a music video was recorded at the Playboy Mansion, its release was scrapped.']], ['Overexposed (album)', ['Overexposed is the fourth studio album by the American band Maroon 5, released on June 20, 2012, by A&M Octone Records.', ' The album was recorded between 2011 and 2012 and was driven after the moderate success of their third album, \"Hands All Over\" (2010), and later by the enormous success of the re-released edition, which was promoted by the single \"Moves Like Jagger\", recorded as a collaboration with Christina Aguilera.', \" It is the band's only album not to feature keyboardist, rhythm guitarist and background vocalist, Jesse Carmichael, who took a temporary break from performing with the group to focus on his studies back then, but eventually returned to Maroon 5 in 2014.\", \" The band worked with different producers, such as Max Martin, who serves as the album's executive producer, Ryan Tedder, Shellback and Benny Blanco.\", \" The band's guitarist and Levine's usual co-writer James Valentine only has four credits on the album.\"]], ['She Will Be Loved', ['\"She Will Be Loved\" is a song by the American pop rock band Maroon 5.', ' The song was written by frontman Adam Levine and lead guitarist James Valentine.', ' It was released as the third single from Maroon 5\\'s debut album, \"Songs About Jane\" (2002).', ' The single peaked at No. 5 in the United States, and by December 2012 had sold more than 2,722,000 digital downloads.', ' It peaked at No. 4 in the United Kingdom.', ' In Australia, it reached No. 1, a position it held for five non-consecutive weeks.', ' The single is noted for its music video starring Kelly Preston in a mother-daughter love triangle with lead singer Adam Levine.', ' The video is also in Sepia.']], ['V (Maroon 5 album)', ['V (Roman numeral for five) is the fifth studio album by American band Maroon 5.', ' It was released on August 29, 2014, by Interscope Records.', ' \"V\" was Maroon 5\\'s first album to be released through Interscope after the band\\'s previous label, A&M Octone Records, transferred them along with most of its artists to Interscope.', ' The album also saw the return of keyboardist/rhythm guitarist/backing vocalist Jesse Carmichael after his absence from recording, touring and promoting \"Overexposed\" (2012).']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.511\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a72830e5542994cef4bc2f9', 'answer': 'San Francisco, California', 'question': 'Where is the company that distributed XXXTentacion\\'s single \"Revenge\" based?', 'supporting_facts': [['Revenge (song)', 0], ['Empire Distribution', 0]], 'context': [['Fortress of Amerikkka', ['Fortress of Amerikkka is a 1989 action film directed by Eric Louzil and distributed by Troma Entertainment.', \" The movie follows John Whitecloud, a criminal rebel who's out for revenge against a corrupt Sheriff and a militia run by a crazed General.\", \" It is often cited by Troma fans as one of the company's worst films.\"]], ['IRIS Distribution', ['IRIS Distribution (Independent Recording Industry Services) - now The Orchard - is a digital music distribution and marketing firm based in San Francisco, CA and New York City.', ' Co-founded by Matt Laszuk (CEO), Bryn Boughton (CMO) and Eric Ferraro (General Counsel) in 2003, IRIS distributes digital music, ringtones, and video to over 450 digital retailers in over 85 countries.', ' It also promotes label catalogs through various outlets, including social networking sites, newsletters, and label profiles.', ' The company offers its technical, marketing, and legal services to a wide range of independent artists and labels from all genres.', ' Artists distributed by IRIS include She Wants Revenge, Le Tigre, Lou Reed, Mogwai, and many more.']], ['Empire Distribution', ['EMPIRE is an American distribution company and record label based in San Francisco, California with offices in New York City and Atlanta.', ' Founded in 2010 by Ghazi Shami, it has released albums in the genres of hip hop, R&B, reggae, rock, gospel, Latin, Country and pop.']], ['The Warped Ones', ['The Warped Ones (狂熱の季節 , Kyōnetsu no kisetsu , aka Season of Heat, Wild Love-Makers and The Weird Lovemakers) is a 1960 Japanese Sun Tribe film directed by Koreyoshi Kurahara and starring Tamio Kawachi, Eiji Go, Yuko Chishiro and Noriko Matsumoto.', ' It was produced and distributed by the Nikkatsu Company.', ' The story concerns the young hoodlum Akira, his friends, their transgressions and specifically their revenge on the couple that got him sent to jail, a reporter and his fiancée.', ' When the fiancée finds herself pregnant by Akira she enlists his help with her finance who has become distant since the attack.']], ['Black Is White', ['Black Is White is a 1920 American silent drama film starring Dorothy Dalton and directed by Charles Giblyn.', ' It was produced by Thomas H. Ince and distributed by Paramount Pictures.', ' The movie is based on a novel, \"Black is White\", by George Barr McCutcheon.', \" The film's spelling differs from the spelling of the novel.\", ' The plot is one in which a woman stands almost any form of abuse from a man and finally forgives him at the moment she has opportunity for the revenge she has always sought, such stories being somewhat popular at the time.']], ['Look at Me (XXXTentacion song)', ['\"Look at Me\" (stylized as \"Look At Me!\")', ' is the debut single by American rapper XXXTentacion.', \" The song premiered on December 30, 2015 on the SoundCloud account of Rojas, the song's co-producer, before initially being released for digital download as a single on January 29, 2016, becoming a sleeper hit after its initial release until January 2017, in which the single was later re-released for digital download again with a remastered and clean version of the single on February 20, 2017, by Empire Distribution.\", ' The song serves as the lead single from his debut commercial mixtape \"Revenge\".', ' The track was produced by Rojas and Jimmy Duval and heavily samples the song \"Changes\" by British dubstep DJ and record producer Mala.']], ['17 (XXXTentacion album)', ['17 is the debut studio album by American rapper XXXTentacion.', ' It was released on August 25, 2017 by Bad Vibes Forever and Empire Distribution.', ' It features 11 tracks and was supported by the lead single \"Revenge.\"', ' \"17\" is X\\'s second solo commercial project, succeeding the compilation mixtape \"Revenge\" (2017).', ' It includes a guest appearance from Trippie Redd and production from X himself, Nick Mira, Taz Taylor, Natra Average, and Potsu.', ' The album experiments with a variety of genres, such as emo, indie rock, and lo-fi.']], ['The First Wives Club', ['The First Wives Club is a 1996 American comedy film, based on the best-selling 1992 novel of the same name by Olivia Goldsmith.', ' Narrated by Diane Keaton, it stars Keaton, Goldie Hawn, and Bette Midler as three divorced women who seek revenge on their ex-husbands who left them for younger women.', ' Stephen Collins, Victor Garber and Dan Hedaya co-star as the husbands, and Sarah Jessica Parker, Marcia Gay Harden and Elizabeth Berkley as their lovers, with Maggie Smith, Bronson Pinchot and Stockard Channing also starring.', ' Scott Rudin produced and Hugh Wilson directed; the film was distributed by Paramount Pictures.']], ['Revenge (song)', ['\"Revenge\", originally titled as \"Garrett\\'s Revenge\" is a single by American recording artist XXXTentacion.', ' The song was released on May 18, 2017 for digital download as a single by Empire Distribution.', ' It is the lead single from his debut studio album \"17\".']], ['Revenge (XXXTentacion album)', ['Revenge is a mixtape by American rapper XXXTentacion, released on May 16, 2017 by Empire Distribution.', \" It consists of 8 previously released songs that were available for streaming on XXXTentacion's SoundCloud.\", ' It was preceded by the lead single \"Look at Me\", which peaked at number 34 on the US \"Billboard\" Hot 100.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.512\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abaca935542996606241610', 'answer': 'Indiana University', 'question': '\"Text Me Merry Christmas\" is a song performed by Kristen Bell and a group that originated at what univeristy?', 'supporting_facts': [['Text Me Merry Christmas', 0], ['Straight No Chaser (group)', 0]], 'context': [['Merry Christmas from the Family', ['\\'\"Merry Christmas from the Family\" is a holiday song written by alternative country artist Robert Earl Keen.', ' It has become extremely popular among the fans within his cult following.', ' The song was first recorded for Keen\\'s 1994 album, \"Gringo Honeymoon\".', ' A live version also appears on his 1996, \"No. 2 Live Dinner\".', ' The popularity of the song led Keen to write a sequel song, \"Happy Holidays Y\\'all\", for his 1998 album \"Walking Distance\", and to publish a book, \"Merry Christmas from the Family\", in 2001.', \" The original song, the book, and the sequel all center around the same cast of characters in Keen's humorous vision of a Texas style Christmas.\"]], ['The Classic Christmas Album (Johnny Mathis album)', ['The Classic Christmas Album is a Christmas compilation album by American pop singer Johnny Mathis that was released on October 7, 2014, by Columbia Records and includes two 1961 recordings that were previously unavailable: \"Ol\\' Kris Kringle\" and the original version of the title track from his 1969 Christmas album \"Give Me Your Love for Christmas\".', ' Three other songs (\"Christmas in the City of the Angels\", \"Sign of the Dove\" and \"The Very First Christmas Day\") make their debut on compact disc as of this release, and two other non-album singles (\"Christmas Is...\" and \"My Kind of Christmas\") can be counted among the rarities here.', ' The collection also includes a selection or two from several of Mathis\\'s Christmas studio albums—\"Sleigh Ride\" from \"Merry Christmas\", \"Have Yourself a Merry Little Christmas\" from \"Sounds of Christmas\", \"Calypso Noel\" from \"Give Me Your Love for Christmas\", \"The Christmas Waltz\" and \"It\\'s Beginning to Look a Lot Like Christmas\" from \"Christmas Eve with Johnny Mathis\", and \"Home for the Holidays\" from \"Sending You a Little Christmas\"—as well as his duet with Bette Midler from her 2006 holiday album \"Cool Yule\", which was a medley of \"Winter Wonderland\" and \"Let It Snow!', ' Let It Snow!', ' Let It Snow!', '\".']], ['Merry Christmas II You', ['Merry Christmas II You is the second Christmas album and thirteenth studio album by American singer and songwriter Mariah Carey.', ' It was released by Island Records on November 2, 2010.', ' Recording began in April 2010 and continued while Carey became pregnant.', ' She was the executive producer of \"Merry Christmas II You\" and worked with various record producers, including Bryan-Michael Cox, Jermaine Dupri, Randy Jackson, James Poyser, Marc Shaiman, James \"Big Jim\" Wright and Johnny \"Sev\" Severin of RedOne.', ' The album features Carey\\'s mother Patricia Carey as a guest vocalist on \"O Come All Ye Faithful\" / \"Hallelujah\\xa0Chorus\".', ' The album is composed of original songs and covers, ballads and uptempo tracks.', ' It incorporates R&B, soul and house music in its composition.']], ['A Merry Christmas!', ['A Merry Christmas!', ' (full title \"From the Creative World of Stan Kenton comes A Merry Christmas!\")', ' is an album of Christmas music by the Stan Kenton Orchestra recorded in 1961 and released by Capitol Records.']], ['A Very Merry Christmas', [\"A Very Merry Christmas is Bobby Vinton's ninth studio album and first Christmas album, released in October 1964.\", ' Vinton had released a four-track Christmas EP which entered the charts the previous year, containing none of the tracks included on \"A Very Merry Christmas\".', ' Due to Billboard editorial policy, it was held off the regular Billboard LP listings.', ' It reached #13 on a Christmas version of the Billboard Hot 200 list of popular albums.', \" The CD, not currently in print, is a valuable collectors' item.\", ' Some of the tracks are available on the in-print disc \"Kissin\\' Christmas\".']], ['Feliz Navidad (song)', ['\"Feliz Navidad\" (] ) is a macaronic Christmas song written in 1970 by the Puerto Rican singer and songwriter José Feliciano.', ' With its simple Spanish chorus (the traditional Christmas/New Year greeting, \"Feliz Navidad, próspero año y felicidad\" meaning \"Merry Christmas, a prosperous year and happiness\") and equally simple English verse \"I wanna wish you a Merry Christmas from the bottom of my heart\", it has become a classic Christmas pop song in the United States, throughout the Spanish-speaking world and internationally.']], ['Text Me Merry Christmas', ['\"Text Me Merry Christmas\" is a song performed by Straight No Chaser and Kristen Bell.', ' It was written by Adam Schlesinger and David Javerbaum.', ' The song was released on November 17, 2014.']], ['Straight No Chaser (group)', ['Straight No Chaser (SNC) is a professional a cappella group which originated in 1996 at Indiana University.', ' In 2007, a 1998 video of \"The 12 Days of Christmas\", went viral with over 8 million views and subsequently led to a five-album record deal with Atlantic Records in 2008.', ' The YouTube video has been viewed over 20 million times.']], ['Merry Christmas (Bing Crosby album)', ['Merry Christmas is a compilation album by Bing Crosby that was released in 1945 on Decca Records.', ' It has remained in print through the vinyl, CD, and downloadable file eras, currently as the disc and digital album \"White Christmas\" on MCA Records, a part of the Universal Music Group, (reissued in June 1995) and currently on vinyl as \"Merry Christmas\" on Geffen Records (re-issued in September 2014).', ' It includes Crosby\\'s signature song \"White Christmas\", the best-selling single of all time with estimated sales of over 50 million copies worldwide.', ' The album has sold over 15 million copies and is the second best-selling Christmas album of all-time behind \"Elvis\\' Christmas Album\", which has sold more than 19 million copies worldwide.']], ['All I Want for Christmas Is You', ['\"All I Want for Christmas Is You\" is a Christmas song performed by American singer and songwriter Mariah Carey.', ' She wrote and produced the song with Walter Afanasieff.', ' Columbia Records released it on November 1, 1994, as the lead single from her first holiday album and fourth studio album, \"Merry Christmas\".', ' \"Christmas\" is an uptempo love song that includes bell chimes and heavy back-up vocals, as well as use of synthesizers.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 0/500 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.513\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab700ca5542991d32223736', 'answer': '\"Now and Then\" (1995)', 'question': 'Which other film did one of the supporting cast in \"Sleepless in Seattle\" appear?', 'supporting_facts': [['Sleepless in Seattle', 1], ['Rita Wilson', 1]], 'context': [['Tom Hanks', ['Thomas Jeffrey Hanks (born July 9, 1956) is an American actor and filmmaker.', ' He is known for his various comedic and dramatic film roles, including \"Splash\" (1984), \"Big\" (1988), \"Turner & Hooch\" (1989), \"A League of Their Own\" (1992), \"Sleepless in Seattle\" (1993), \"Philadelphia\" (1993), \"Forrest Gump\" (1994), \"Apollo 13\" (1995), \"Saving Private Ryan\" (1998), \"You\\'ve Got Mail\" (1998), \"The Green Mile\" (1999), \"Cast Away\" (2000), \"Road to Perdition\" (2002), and \"The Da Vinci Code\" (2006), as well as for his voice work in the animated films \"The Polar Express\" (2004) and the \"Toy Story\" series.']], ['List of Waterloo Road characters (series 8)', ['The following is a list of characters who appear in the eighth series of the BBC school drama \"Waterloo Road\", in order of appearance.', ' The Eighth Series consists of Thirty Episodes, first broadcast from 23 August 2012 to 4 July 2013.', ' New Main Cast Members from Episode One include Head of English (later Head Teacher) Christine Mulgrew and History Teacher Audrey McFall, with Maggie Croft (later Budgen) and Lorraine Donnegan also promoted to the Main Cast.', \" New Pupil Characters from Episode One include Christine's son Connor Mulgrew, Imogen Stewart, Jade Fleming, Lula Tsibi, Rhiannon Salt and Angus Hancock.\", ' Head of rival school Havelock High Gerald Findlay also appears in the first five episodes.', \" Episode four sees the first of several appearances of Imogen's mother Sally Stewart; Michael Byrne's father Billy debuts in the same episode, and appears until his death in Episode Ten.\", ' Pupil Liberty Gordon first appears in Episode Five, and Kevin Skelton (later Chalk) also joins the supporting cast in Episode Eight.', \" Lorraine's sister Sonya Donnegan joins the main cast as School Secretary from Episode Nine.\", ' Episode Eleven sees the Barry family, consisting of mother Carol and her children Barry, Dynasty and Kacey all join the supporting cast, with pupil Jack MacAllister also debuting in the same episode.', \" Towards the end of the series, supporting characters of Maintenance Assistant Ndale Kayuni, Dynasty's former boyfriend Steve-O Malone and Acting Head of Science Esther Fairclough all appear.\", ' Angus Deayton and Richard Mylan join the cast as Head of Modern Languages George Windsor and Deputy Head Simon Lowsley in episodes Twenty-Seven and Twenty-Nine respectively.', ' Recurring Character Robert Bain, Head of Greenock Education makes his first appearance in episode Twenty-Seven, and wife of George Windsor, Princess first appears in episode Twenty-Nine.']], ['List of Waterloo Road characters (series 9)', ['The following is a list of characters who first appear in the ninth series of the BBC school drama \"Waterloo Road\", in order of first appearance.', ' The ninth series consists of twenty episodes, first broadcast from 5 September 2013 to 12 March 2014.', ' With the majority of new cast members already introduced in Series Eight, Science Teacher Sue Lowsley completes the main cast in episode one.', ' New pupil characters include twins Lenny and Lisa Brown, Darren Hughes, Shaznay Montrose and Archie Wong.', \" Episode ten sees the first appearance of Sue's sister Vix Spark, who continues in a supporting role for the rest of the series.\", ' Episode eleven introduces PE teacher Hector Reid to the main cast, and pupil Gabriella Wark to the supporting cast.']], ['The Ben Maller Show', ['The Ben Maller Show is a sports radio talk show that currently airs on Fox Sports Radio during the hours of 2:00 AM to 6:00 AM ET that is hosted by the Legendary Ben Maller.', ' The current edition of the Ben Maller Show debuted on January 6, 2014, after the announcement of a shake-up in the on-air broadcasting lineup.', \" It was announced that with JT The Brick's promotion, the supporting cast of his former slot would be staying on the time slot and thus joining Maller.\", ' The supporting cast consists of Eddie Garcia (update anchor/sidekick), Justin Cooper (Coopa-loop) (executive producer) and Danny G. (technical producer).', ' Maller\\'s previous producer Justin Cooper would be staying with Maller\\'s old slot and thus working with Brian Noe and \"The Noe Show\" which currently airs in Maller\\'s time slot during the weekend.']], ['Gaby Hoffmann', ['Gabriella Mary \"Gaby\" Hoffmann (born January 8, 1982) is an American film and television actress best known for her roles on \"Sleepless in Seattle,\" \"Transparent\" and \"Girls\",<ref name=\"Vulture-Girls/Trans-2014\"> </ref> which garnered her nominations for the Primetime Emmy Award for Outstanding Supporting Actress in a Comedy Series and Primetime Emmy Award for Outstanding Guest Actress in a Comedy Series in 2015, respectively.', ' Additionally, she is remembered as a child actress from the films \"Field of Dreams\", \"Uncle Buck\", \" Now and Then\", and \" Volcano\".']], ['List of Waterloo Road characters (series 6)', ['The following is a list of characters who first appear in the sixth series of the BBC school drama \"Waterloo Road\", in order of first appearance.', ' The sixth series consists of twenty episodes, first broadcast from 1 September 2010 to 6 April 2011.', \" The series opens with the introduction of new head teacher Karen Fisher; other additions to the main cast include Karen's husband Charlie, head of Spanish Francesca Montoya and geography teacher Marcus Kirby, as well as returning character Janeece Bryant.\", \" The Fishers' children Bex Fisher, Jess and Harry, Marcus' children Jonah and Ruth, and Ronan Burley all join the supporting cast as pupils from episode one, alongside Vicky MacDonald who returns to the series in a regular role.\", ' Episode eleven sees head of pastoral care Adanna Lawal join the main cast, and pupils Kyle Stack and Nate Gurney make their first appearances in the same episode.']], ['The Only Way Is Essex (series 8)', ['The eighth series of \"The Only Way Is Essex\", a British semi-reality television programme, began airing on 24 February 2013 on ITV2.', ' The series consisted of 12 episodes.', ' Series 8 marked the first series not to feature Lydia Bright, after her departure from the series in December 2012.', ' The series also saw the departure of numerous supporting cast members and introduced new supporting cast members.']], ['The High and the Mighty (film)', ['The High and the Mighty is a 1954 \"WarnerColor\" American \"disaster\" film in CinemaScope directed by William A. Wellman and written by Ernest K. Gann who also wrote the 1953 novel on which his screenplay was based.', \" The film's cast was headlined by John Wayne, who was also the project's co-producer.\", ' Composer Dimitri Tiomkin won an Academy Award for his original score while his title song for the film also was nominated for an Oscar (although the title song did not actually appear in release prints nor in the recent restoration of the film).', ' The film received mostly positive reviews and grossed $8.5 million in its theatrical release.', ' The supporting cast includes Claire Trevor, Laraine Day, Robert Stack, Jan Sterling, Phil Harris and Robert Newton.']], ['Rita Wilson', ['Rita Wilson (born Margarita Ibrahimoff; October 26, 1956) is an American actress, singer, voice actress, activist, and producer.', ' She appeared in the films \"Sleepless in Seattle\" (1993), \"Now and Then\" (1995), \"Jingle All the Way\" (1996), \"The Story of Us\" (1999) and \"Runaway Bride\" (1999).', ' Wilson has also performed on Broadway and on television, and she has produced several films, including \"My Big Fat Greek Wedding\" (2002).']], ['Sleepless in Seattle', ['Sleepless in Seattle is a 1993 American romantic comedy-drama film directed and co-written by Nora Ephron, based on a story by Jeff Arch.', \" It stars Tom Hanks and Meg Ryan, alongside a supporting cast featuring Bill Pullman, Ross Malinger, Rob Reiner, Rosie O'Donnell, Gaby Hoffmann, Victor Garber, and Rita Wilson.\", ' The film was a critical and commercial success, grossing over $220 million worldwide.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.514\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae0a59a55429945ae9593e2', 'answer': 'Roman', 'question': 'This Celtic ruler who was born in AD 43 ruled southeastern Britain prior to conquest by which empire?', 'supporting_facts': [['Togodumnus', 0], ['Catuvellauni', 0]], 'context': [['Romano-British culture', ['Romano-British culture is the culture that arose in Britain under the Roman Empire following the Roman conquest in AD 43 and the creation of the province of Britannia.', ' It arose as a fusion of the imported Roman culture with that of the indigenous Britons, a people of Celtic language and custom.', ' It survived the 5th century Roman departure from Britain.', ' Scholars such as Christopher Snyder believe that during the 5th and 6th centuries – approximately from AD 410 when the Roman legions withdrew, to AD 597 when St Augustine of Canterbury arrived – southern Britain preserved an active sub-Roman culture that survived the attacks from the Anglo-Saxons and even used a vernacular Latin when writing.']], ['History of Wales', ['The history of Wales begins with the arrival of human beings in the region thousands of years ago.', ' Neanderthals lived in what is now Wales, or \"Cymru\" in Welsh, at least 230,000 years ago, while \"Homo sapiens\" arrived by about 31,000 BC.', ' However, continuous habitation by modern humans dates from the period after the end of the last ice age around 9000 BC, and Wales has many remains from the Mesolithic, Neolithic, and Bronze Age.', ' During the Iron Age the region, like all of Britain south of the Firth of Forth, was dominated by the Celtic Britons and the Brittonic language.', ' The Romans, who began their conquest of Britain in AD 43, first campaigned in what is now northeast Wales in 48 against the Deceangli, and gained total control of the region with their defeat of the Ordovices in 79.', ' The Romans departed from Britain in the 5th century, opening the door for the Anglo-Saxon invasion.', ' Thereafter Brittonic language and culture began to splinter, and several distinct groups formed.', ' The Welsh people were the largest of these groups, and are generally discussed independently of the other surviving Brittonic-speaking peoples after the 11th century.']], ['Durotriges', ['The Durotriges were one of the Celtic tribes living in Britain prior to the Roman invasion.', ' The tribe lived in modern Dorset, south Wiltshire, south Somerset and Devon east of the River Axe and the discovery of an Iron Age hoard in 2009 at Shalfleet, Isle of Wight gives evidence that they lived in the western half of the island.', ' After the Roman conquest, their main \"civitates\", or settlement-centred administrative units, were Durnovaria (modern Dorchester, \"the probable original capital\") and Lindinis (modern Ilchester, \"whose former, unknown status was thereby enhanced\").', ' Their territory was bordered to the west by the Dumnonii; and to the east by the Belgae.']], ['Corieltauvi', ['The Corieltauvi (formerly thought to be called the Coritani, and sometimes referred to as the Corieltavi) were a tribe of people living in Britain prior to the Roman conquest, and thereafter a \"civitas\" of Roman Britain.', ' Their territory was in what is now the English East Midlands.', ' They were bordered by the Brigantes to the north, the Cornovii to the west, the Dobunni and Catuvellauni to the south, and the Iceni to the east.', ' Their capital was called \"Ratae Corieltauvorum\", known today as Leicester.']], ['AD 43', ['AD 43 (XLIII) was a common year starting on Tuesday (link will display the full calendar) of the Julian calendar.', ' At the time, it was known as the Year of the Consulship of Caesar and Vitellius (or, less frequently, year 796 \"Ab urbe condita\").', ' The denomination AD 43 for this year has been used since the early medieval period, when the Anno Domini calendar era became the prevalent method in Europe for naming years.']], ['Togodumnus', ['Togodumnus (d. AD 43) was a historical king of the British Catuvellauni tribe at the time of the Roman conquest.', ' He can probably be identified with the legendary British king Guiderius.']], ['Roman conquest of Britain', ['The Roman conquest of Britain was a gradual process, beginning effectively in AD 43 under Emperor Claudius, whose general Aulus Plautius served as first governor of Roman Britain (Latin: \"Britannia\" ).', ' Great Britain had already frequently been the target of invasions, planned and actual, by forces of the Roman Republic and Roman Empire.', \" In common with other regions on the edge of the empire, Britain had enjoyed diplomatic and trading links with the Romans in the century since Julius Caesar's expeditions in 55 and 54 BC, and Roman economic and cultural influence was a significant part of the British late pre-Roman Iron Age, especially in the south.\"]], ['Sussex chicken', ['The Sussex chicken is a dual purpose breed of chicken that originated in England around the time of the Roman conquest of Britain in AD 43 that is a popular garden chicken in many countries.', ' They come in eight colours (with a couple more being developed) and have a bantam version at 1/4 size; the bantams may be any of the eight colours.']], ['Catuvellauni', ['The Catuvellauni were a Celtic tribe or state of southeastern Britain before the Roman conquest, attested by inscriptions into the 4th century.']], ['History of England', ['England became inhabited more than 800,000 years ago, as the discovery of flint tools and footprints at Happisburgh in Norfolk has revealed.', ' The earliest evidence for early modern humans in North West Europe, a jawbone discovered in Devon at Kents Cavern in 1927, was re-dated in 2011 to between 41,000 and 44,000 years old.', ' Continuous human habitation in England dates to around 13,000 years ago (see Creswellian), at the end of the last glacial period.', ' The region has numerous remains from the Mesolithic, Neolithic, and Bronze Age, such as Stonehenge and Avebury.', ' In the Iron Age, England, like all of Britain south of the Firth of Forth, was inhabited by the Celtic people known as the Britons, including some Belgic tribes (e.g. the Atrebates, the Catuvellauni, the Trinovantes, etc.) in the south east.', ' In AD 43 the Roman conquest of Britain began; the Romans maintained control of their province of Britannia until the early 5th century.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.515\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae4d3a55542990ba0bbb164', 'answer': 'Dewey Lake Monster', 'question': 'What creature of American folklore gained notoriety in 1964?', 'supporting_facts': [['Dewey Lake Monster', 0], ['Dewey Lake Monster', 1], ['Bigfoot', 0]], 'context': [['Bigfoot', ['Bigfoot (also known as Sasquatch) is a cryptid which supposedly is a simian-like creature of American folklore that is said to inhabit forests, especially in the Pacific Northwest.', ' Bigfoot is usually described as a large, hairy, bipedal humanoid.', ' The term \"sasquatch\" is an Anglicized derivative of the Halkomelem word \"sásq\\'ets\".']], ['Hidebehind', ['The Hidebehind is a nocturnal fearsome critter from American folklore that preys upon humans that wander the woods, and was credited for the disappearances of early loggers when they failed to return to camp.', ' As its name suggests, the Hidebehind is noted for its ability to conceal itself.', \" When an observer attempts to look directly at it, the creature hides again behind an object or the observer and therefore can't be directly seen: a feat it accomplishes by sucking in its stomach to a point where it is so slender that it can easily cover itself behind the trunk of any tree.\", ' The Hidebehind uses this ability to stalk human prey without being observed and to attack without warning.', \" Their victims, including lumberjacks who frequent the forests, are dragged back to the creature's lair to be devoured.\", ' The creature subsists chiefly upon the intestines of its victim, and has a severe aversion to alcohol, which is considered a sufficient repellent.', ' Tales of the Hidebehind may have helped explain strange noises in the forest at night.', ' Early accounts describe hidebehinds as large, powerful animals, despite the fact that no one was able to see them.']], ['Chessie (sea monster)', ['In American folklore, Chessie is a sea monster said to live in the midst of the Chesapeake Bay.', ' Over the years there have been many alleged sightings of a serpent-like creature with flippers as part of its body.', ' Most sighting reports describe it as a long, snake-like creature, from 25 ft to 40 ft long.', ' It is said to swim using its body as a sine curve moving through the water.', ' There were a rash of sightings in 1977 and more in the 1980s, with occasional reports since then.']], ['Dewey Lake Monster', ['The Dewey Lake Monster is the name given to a large bipedal creature approximately 10 feet (3 meters) tall and weighing about 500 pounds (227 kilograms), which first gained wide notoriety in June 1964 after several reported sightings near Dewey Lake in Dowagiac, Michigan.', ' It is also referred to as the Michigan Bigfoot and Sister Lakes Sasquatch.']], ['Elsie Clews Parsons', ['Elsie Worthington Clews Parsons (November 27, 1875 – December 19, 1941) was an American anthropologist, sociologist, folklorist, and feminist who studied Native American tribes—such as the Tewa and Hopi—in Arizona, New Mexico, and Mexico.', ' She helped found The New School.', ' She was associate editor for \"The Journal of American Folklore\" (1918-1941), president of the American Folklore Society (1919-1920), president of the American Ethnological Society (1923-1925), and was elected the first female president of the American Anthropological Association (1941) right before her death.']], ['American Folklore Society', ['The American Folklore Society (AFS) is the US-based professional association for folklorists, with members from the US, Canada, and around the world, which aims to encourage research, aid in disseminating that research, promote the responsible application of that research, publish various forms of publications, advocate for the continued study and teaching of folklore, etc.', ' The Society is based at Indiana University and has an annual meeting every October.', ' The Society\\'s quarterly publication is the \"Journal of American Folklore\".', ' The current president is Kay Turner, independent scholar and public folklorist.']], ['Northern Sky Theater', ['Northern Sky Theater (formerly known as American Folklore Theatre or AFT) is a professional theater company that creates, develops, and produces musicals based on the populist culture and heritage of the United States.', ' Located in Door County, Wisconsin, the company began in 1970 as The Heritage Ensemble, performing on the stage of the 700-seat outdoor theater in Peninsula State Park.', ' In 1990, American Folklore Theatre was co-founded by Fred Alley, Frederick Heide, and Gerald Pelrine.', ' In 2015, the company changed its name to Northern Sky Theater.']], ['Levette J. Davidson', ['Levette J. Davidson was a nationally acclaimed expert in folklore, especially that of Colorado and the West.', ' He was born in Eureka, Illinois May 16, 1894, one of four children.', ' Because his grand uncle was past-President of Eureka College, a Christian seminary, Davidson was \"reared in the school\\'s shadow\" with the option of becoming \"either a teacher or a preacher.\"', ' He chose teaching and was awarded his B.A. from Eureka in 1915.', ' A year later he received his A.M. degree from the University of Illinois where he received Phi Beta Kappa honors.', ' In 1917 he earned his M.A. in social science and history at Harvard University.', ' __During World War I, he served with the Tenth Infantry of the Forty-Sixth Division and also served as an army sergeant in Intelligence.', ' Davidson and his wife Mary, also a graduate of Eureka, were married in 1918.', ' At the end of the war, Davidson simultaneously taught and studied at the University of Michigan where he earned a Ph.D. in languages and literatures in 1922.', ' He arrived in Denver later that year and began teaching at the University of Denver.', ' Before arriving in Colorado, his academic specialty was eighteenth century England.', ' Once in Denver, he realized that Western literature and folklore would be his life work.', ' Davidson taught at the University of Denver until his death in 1957.', ' His course topics ranged from studies of Shakespeare and other English literature, to folklore in the West.', ' He began teaching news writing and eventually founded the journalism department at the University as well as the board of publications and the press club.', ' He was the first faculty member elected to serve as president of the University Senate.', ' In 1940 he became head of the English Department and from March to August 1953 he served as interim Chancellor.', ' At the time of assuming his post, he remarked: \"There is hardly an organization on campus that I have not been connected with at one time or another.\"', ' Davidson was named University Lecturer for 1956.', ' Davidson was a director of the Colorado State Historical Society, the Modern Languages Association, the American Folklore Society, and the American Dialect Society.', ' He was a charter member of the Denver Posse of the Westerners, an organization devoted to western folklore.', ' In spare moments, Davidson conducted research, wrote outlines for plays, and authored a large number of articles.', ' He contributed to a wide range of magazines from \"Western American\" to \"Shakespeare Quarterly\".', ' Additionally, Davidson authored several books on folklore.', ' The most widely recognized is \"Rocky Mountain Tales\", which presents regional folklore as well as true accounts of early events in the area.', ' Levette Davidson died May 14, 1957.']], ['Teakettler', ['The Teakettler (Urocyon iugulebesonia) is a legendary creature from American folklore with origins in lumberjack culture, specifically the lumber camps of Minnesota and Wisconsin.', ' It is part of a group of similar folklore creatures known collectively as Fearsome Critters.', ' It is said to resemble a small stubby legged dog with the ears of a cat.', ' Its name comes from the sound it makes, which is akin to that of a boiling tea kettle.', ' It only walks backwards, and steam issues from its mouth as it makes its whistle.', ' As the myth goes, only a few lumberjacks have seen one, as they are very shy, but if a boiling kettle is heard and nowhere to be found, it is sure that a Teakettler is nearby.']], ['Journal of American Folklore', ['The Journal of American Folklore is a peer-reviewed academic journal published by the American Folklore Society.', ' Since 2003 this has been done on its behalf by the University of Illinois Press.', \" The journal has been published since the society's founding in 1888.\", ' It publishes on a quarterly schedule and incorporates scholarly articles, essays, and notes relating to its field.', ' It also includes reviews of books, exhibitions and events.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.515\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8bd49d5542997f31a41dd7', 'answer': 'Matt Groening', 'question': 'Who is this American cartoonist, writer, producer, animator, and voice actor that worked with this multiple Shuster Award, Harvey Award and Eisner Award nominee and an Eisner Award–winning comic book creator?', 'supporting_facts': [['Ian Boothby', 0], ['Matt Groening', 0]], 'context': [['Ross Richie', ['Ross Richie (born May 22, 1970) is an American comic book publisher and the founder of Boom!', ' Studios, film producer, television producer and comic book creator.', ' Richie has written guest columns for The Hollywood Reporter been a Keynote Speaker for The Harvey Awards and a judge for the \"Spirit of Comics Retailer\" Eisner Award.', ' The New York Times profiled Richie and his company Boom!', ' Studios twice.', ' 20th Century Fox production president Emma Watts called Richie\\'s company \"a publishing powerhouse devoted to original, innovative storytelling and world-class artists\" and Deadline.com named it an \"IP mega-library\".', ' The Wall Street Journal cited the Boom!', ' Studios movie Richie produced, 2 Guns starring Denzel Washington and Mark Wahlberg, a \"successful adaptation\\' and featured Boom!', ' Studios in their article on the 2016 San Diego Comic Con International.', \" Richie is a five-time publisher of New York Times best-sellers Mouse Guard the Adventure Time graphic novels Lumberjanes, Clive Barker's Hellraiser graphic novels and Irredeemable.\"]], ['Matt Groening', ['Matthew Abraham \"Matt\" Groening ( ; born February 15, 1954) is an American cartoonist, writer, producer, animator, and voice actor.', ' He is the creator of the comic strip \"Life in Hell\" (1977–2012) and the television series \"The Simpsons\" (1989–present), \"Futurama\" (1999–2003, 2008–2013), and the upcoming \"Disenchantment\" (2018).', ' \"The Simpsons\" has gone on to become the longest-running U.S. primetime-television series in history, as well as the longest-running animated series and sitcom.']], ['Will Eisner', ['William Erwin \"Will\" Eisner ( ; March 6, 1917\\xa0– January 3, 2005) was an American cartoonist, writer, and entrepreneur.', ' He was one of the earliest cartoonists to work in the American comic book industry, and his series \"The Spirit\" (1940–1952) was noted for its experiments in content and form.', ' In 1978, he popularized the term \"graphic novel\" with the publication of his book \"A Contract with God\".', ' He was an early contributor to formal comics studies with his book \"Comics and Sequential Art\" (1985).', ' The Eisner Award was named in his honor, and is given to recognize achievements each year in the comics medium; he was one of the three inaugural inductees to the Will Eisner Comic Book Hall of Fame.']], ['Chris Samnee', ['Chris Samnee is an American comic book artist.', ' He received the 2011 Harvey Award for Most Promising Newcomer for his work on the \"\", and won a 2013 Eisner Award for Best Penciller/Inker for his work on the \"\" and his \"Daredevil\".', ' He was also nominated for the 2006 Russ Manning Most Promising Newcomer Award.']], ['American Born Chinese', ['American Born Chinese is a graphic novel by Gene Luen Yang.', \" Released in 2006 by First Second Books, it was a finalist for the 2006 National Book Awards in the category of Young People's Literature.\", ' It won the 2007 Michael L. Printz Award, the 2007 Eisner Award for Best Graphic Album: New, the \"Publishers Weekly\" Comics Week Best Comic of the Year, the \"San Francisco Chronicle\" Best Book of the Year, the 2006/2007 Best Book Award from The Chinese American Librarians Association, and Amazon.com Best Graphic Novel/Comic of the Year.', ' It also made the \"Booklist\" Top Ten Graphic Novel for Youth, the NPR Holiday Pick, and \"Time (Magazine)\" Top Ten Comic of the Year.', ' It was colored by cartoonist Lark Pien, who received the 2007 Harvey Award for Best Colorist for her work on the book.']], ['Ian Boothby', ['Ian Boothby is a multiple Shuster Award, Harvey Award and Eisner Award nominee and an Eisner Award–winning comic book creator best known for his work as the lead writer on \"Simpsons Comics\" and \"Futurama Comics\" for Matt Groening\\'s Bongo Comics.', ' Boothby has written more \"Simpsons Comics\" than any other writer.', ' He has also worked on various Canadian television series and is a well known stand-up, sketch and improv comedian working in the Vancouver area.', ' He co-created Free Willie Shakespeare for the Vancouver Theatresports League which won the Jessie Richardson Theatre Award for Excellence in Interactive Theatre.']], ['Vera Brosgol', ['Vera Brosgol (born August 1984 in Moscow, Russia) is an Eisner Award and Harvey Award winning cartoonist and a graduate in Classical Animation of Sheridan College in Canada.', ' She lives in Portland, Oregon and worked for Laika Entertainment where she did storyboards and concept art for their animation productions.', ' Brosgol has also collaborated with Shaenon Garrity on \"L\\'il Mell and Sergio\" for Girlamatic and drawn several guest comics for John Allison\\'s \"Scary Go Round\".']], ['Steve Purcell', ['Steven Ross Purcell (born 1961) is an American cartoonist, animator, director and game designer.', ' He is most widely known as the creator of \"Sam & Max\", an independent comic book series about a pair of anthropomorphic animal vigilantes and private investigators, for which Purcell received an Eisner Award in 2007.', ' Since being a comic, the series has grown to incorporate an animated television series and several video games.', ' A graduate of the California College of Arts and Craft, Purcell began his career creating comic strips for the college newsletter.', ' He performed freelance work for Marvel Comics and Fishwrap Productions before publishing his first \"Sam & Max\" comic in 1987.', \" Purcell was hired by LucasArts as an artist and animator in 1988, working on several titles within the company's adventure games era.\"]], ['Chuck BB', ['Chuck BB (born 1981 in California) is an Eisner Award winning American comic book creator, best known for his work on \"Spider-Man\", \"Fear Agent\", and Oni\\'s \"Black Metal\".']], ['100 Bullets', ['100 Bullets is an American comic book published by DC Comics under its Vertigo imprint.', ' Written by Brian Azzarello and illustrated by Eduardo Risso, the comic book ran for 100 issues and won the Eisner Award and Harvey Award.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.515\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab29346554299545a2cf997', 'answer': 'KXII', 'question': 'What CBS-affiliated station serves Pontotoc County, Oklahoma?', 'supporting_facts': [['KXII', 0], ['Ada, Oklahoma', 0]], 'context': [['Pontotoc County, Oklahoma', ['Pontotoc County is in the south central part of Oklahoma.', ' As of the 2010 census, the population was 37,492.', ' Its county seat is Ada.', ' The county was created at statehood from part of the Chickasaw Nation in Indian Territory.', ' It was named for a historic Chickasaw tribal area in Mississippi.', ' According to the \"Encyclopedia of Oklahoma History and Culture\", Pontotoc is usually translated \"cattail prairie\" or \"land of hanging grapes.\"']], ['WGCL-TV', ['WGCL-TV, virtual channel 46 (UHF digital channel 19), is a CBS-affiliated television station license to Atlanta, Georgia, United States.', ' The station is owned by the Meredith Corporation as part of a duopoly with independent station WPCH-TV (channel 17).', \" The two stations share a studio located on 14th Street in northwestern Atlanta; WGCL-TV's transmitter is located near North Druid Hills.\", ' WGCL-TV is the third-largest CBS-affiliated station by market size (WUSA in Washington, D.C. being the largest and KHOU in Houston being the second largest) that is not owned and operated by the network.']], ['KXII', ['KXII, virtual channel and VHF digital channel 12, is a CBS-affiliated television station serving the Ada–Sherman media market that is licensed to Sherman, Texas, United States.', ' The station – which also maintains subchannel-only affiliations with MyNetworkTV and Fox – is owned by Gray Television.', ' KXII maintains primary studio facilities located on Texoma Parkway (S.H. 91) in northeastern Sherman; secondary studios are located on South Commerce Street (U.S. Route 77) and Elks Boulevard in southwestern Ardmore, Oklahoma.', ' The station maintains transmitter facilities located along Oklahoma State Highway 99 in rural northeastern Marshall County, Oklahoma (southwest of Madill).', \" KXII's signal is relayed on low-power translator station KXIP-LD (channel 12) in Paris, Texas.\"]], ['Ada, Oklahoma', ['Ada is a city in and the county seat of Pontotoc County, Oklahoma, United States.', ' The population was 16,810 at the 2010 census, an increase of 7.1 percent from 15,691 at the 2000 census.', ' The city was named for Ada Reed, the daughter of an early settler, and was incorporated in 1901.', ' Ada is home to East Central University, and is the headquarters of the Chickasaw Nation.']], ['Pontotoc County, Mississippi', ['Pontotoc County is a county located in the U.S. state of Mississippi.', ' As of the 2010 census, the population was 29,957.', ' Its county seat is Pontotoc.', ' It was created on February 9, 1836 from lands ceded to the United States under the Chickasaw Cession.', ' Pontotoc is a Chickasaw word meaning \"land of hanging grapes\".', ' The original Natchez Trace and the current-day Natchez Trace Parkway both pass through the southeast corner of Pontotoc County.']], ['Ada Municipal Airport', ['Ada Municipal Airport (IATA: ADT,\\xa0ICAO: KADH,\\xa0FAA LID: ADH) is a public airport located two miles (3 km) north of the central business district of Ada, a city in Pontotoc County, Oklahoma, United States.', ' It is owned by the City of Ada, which is located in southeast Oklahoma, 88 mi southeast of Oklahoma City.']], ['Pontotoc, Oklahoma', ['Pontotoc is an unincorporated community in Johnston County, Oklahoma.', ' A post office was established in Pontotoc in 1858.', ' The town was named after Pontotoc County, which was one of the divisions of Chickasaw Nation.']], ['Pontotoc, Mississippi', ['Pontotoc is a city in, and the county seat of, Pontotoc County, Mississippi, located to the west of the much larger city of Tupelo.', ' The population was 5,625 at the 2010 census.']], ['Pontotoc County School District', ['The Pontotoc County School District is a public school district based in Pontotoc County, Mississippi (USA).']], ['Muddy Boggy Creek', ['Muddy Boggy River, also known as the Muddy Boggy Creek, is a 175 mi river in south central Oklahoma.', ' a major tributary of the Red River in south central Oklahoma, is formed by the confluence of Muddy Boggy Creek and Clear Boggy Creek.', ' Both streams converge at a location known as River Mile 24 in Pontotoc County, Oklahoma.', ' It is a major tributary of the Red River.', ' The river is inhabited by over one hundred species of fish.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.516\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab3ede755429976abd1bcf4', 'answer': 'John Ford', 'question': 'Who directed the 1940 film in which John Arledge appeared?', 'supporting_facts': [['John Arledge', 0], ['John Arledge', 1], ['The Grapes of Wrath (film)', 0]], 'context': [['Prison Nurse', ['Prison Nurse is a 1938 American drama film directed by James Cruze and written by Earl Felton and Sidney Salkow.', ' The film stars Henry Wilcoxon, Marian Marsh, Bernadene Hayes, Ben Welden, Ray Mayer and John Arledge.', ' The film was released on March 1, 1938, by Republic Pictures.']], [\"Olsen's Big Moment\", [\"Olsen's Big Moment is a 1933 American comedy film directed by Malcolm St. Clair and written by Henry Johnson and James J. Tynan.\", ' The film stars El Brendel, Walter Catlett, Barbara Weeks, Susan Fleming, John Arledge and Joe Sawyer.', ' The film was released on November 17, 1933, by Fox Film Corporation.']], ['The Spider (1931 film)', ['The Spider is a 1931 American pre-Code mystery film directed by Kenneth MacKenna and William Cameron Menzies and written by Barry Conners.', ' The film stars Edmund Lowe, Lois Moran, El Brendel, John Arledge, George E. Stone and Earle Foxe.', ' The film was released on September 27, 1931, by Fox Film Corporation.']], ['Mexican Spitfire Out West', ['Mexican Spitfire Out West is a 1940 American comedy film directed by Leslie Goodwins and written by Charles E. Roberts and Jack Townley.', ' It is the sequel to the 1940 film \"Mexican Spitfire\" and the second of the film series.', ' The film stars Lupe Vélez, Leon Errol, Donald Woods, Elisabeth Risdon and Cecil Kellaway.', ' The film was released on November 29, 1940, by RKO Pictures.']], ['John Arledge', ['John Arledge (March 12, 1906 – May 15, 1947) was an American film and stage actor.', ' He played dozens of supporting roles in the Hollywood movies of the 1930s–1940s, including \"The Grapes of Wrath\".']], ['County Fair (1937 film)', ['County Fair is a 1937 American drama film directed by Howard Bretherton and starring John Arledge, Mary Lawrence and J. Farrell MacDonald.', ' It was a remake of the 1932 film \"The County Fair\".']], ['Shipmates Forever', ['Shipmates Forever is a 1935 American musical film directed by Frank Borzage and written by Delmer Daves.', ' Set at the United States Naval Academy, the film stars Dick Powell, Ruby Keeler, Lewis Stone, Ross Alexander, John Arledge, Eddie Acuff and Dick Foran.', ' The film was released by Warner Bros. on October 12, 1935.']], ['The Grapes of Wrath (film)', ['The Grapes of Wrath is a 1940 drama film directed by John Ford.', \" It was based on John Steinbeck's 1939 Pulitzer Prize-winning novel of the same name.\", ' The screenplay was written by Nunnally Johnson and the executive producer was Darryl F. Zanuck.']], ['Two in Revolt', ['Two in Revolt is a 1936 American drama film directed by Glenn Tryon.', ' Released on April 3, 1936, by RKO Radio Pictures, the film stars John Arledge, Louise Latimer, and Moroni Olsen, and features Lightning the dog and Warrior the horse.']], ['He Married His Wife', [\"He Married His Wife is a 1940 film about a race horse owner (Joel McCrea) who wants his ex-wife (Nancy Kelly) to remarry so he'll no longer have to pay alimony.\", \" This movie is a black-and-white comedy released 19 January 1940, directed by Roy Del Ruth and written by John O'Hara, among others.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.516\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a776fc15542997042120a3a', 'answer': 'The Bears', 'question': 'What is the mascot of the oldest private university in Georgia?', 'supporting_facts': [[\"2012–13 Mercer Bears men's basketball team\", 0], [\"2012–13 Mercer Bears men's basketball team\", 1], ['Mercer University', 0]], 'context': [['University of Deusto', ['The University of Deusto (Spanish: \"Universidad de Deusto\" ; Basque: \"Deustuko Unibertsitatea\" ) is a Spanish private university owned by the Society of Jesus, with campuses in Bilbao and San Sebastián, and the Deusto Business School branch in Madrid.', ' The University of Deusto is the oldest private university in Spain.']], ['Universidade Cândido Mendes', ['Universidade Cândido Mendes is a private university located in Rio de Janeiro, Brazil.', \" It is Latin America's oldest private university.\"]], ['Islamic University of Indonesia', ['The Islamic University of Indonesia (Indonesian: \"Universitas Islam Indonesia\" or UII , Arabic: الجمعة الاسلامية الاندونيسية) is a private university in Yogyakarta, Indonesia.', ' It was established on 27 Rajab 1364 (Islamic calendar) or on 8 July 1945 as STI (Sekolah Tinggi Islam - Islamic Higher School) by political figures of the day including Dr. Muhammad Hatta, Mohammad Natsir, Mohammad Roem, Wahid Hasyim, and Abdul Kahar Muzakkar.', ' STI developed into a university called Universitas Islam Indonesia on 14 December 1947.', ' Historically, UII is the first national university in Indonesia, and it is the oldest private university in the country.']], ['Emory University', ['Emory University is a private research university in metropolitan Atlanta, located in the Druid Hills section of DeKalb County, Georgia, United States.', ' The university was founded as Emory College in 1836 in Oxford, Georgia by the Methodist Episcopal Church and was named in honor of Methodist bishop John Emory.', ' In 1915, the college relocated to metropolitan Atlanta and was rechartered as Emory University.', ' The university is the second-oldest private institution of higher education in Georgia and among the fifty oldest private universities in the United States.', \" Emory is frequently cited as one of the world's leading research universities and one of the top institutions in the United States.\"]], ['Doshisha University', ['Doshisha University (同志社大学 , Dōshisha daigaku ) , also referred to as Dodai (同大 , Dōdai ) , is a private university in Kyoto City, Japan.', \" Established in 1875, it is one of Japan's oldest private institutions of higher learning, and has approximately 30,000 students enrolled on four different campuses in Kyoto.\", ' It is one of the Japanese \"Global 30\" universities and one of the , a group of the four leading private universities in western Japan\\'s Kansai region.']], ['Norwich University', ['Norwich University – The Military College of Vermont is a private university located in Northfield, Vermont.', ' It is the oldest private military college in the United States.', ' The university was founded in 1819 at Norwich, Vermont, as the American Literary, Scientific and Military Academy.', ' It is the oldest of six senior military colleges, and is recognized by the United States Department of Defense as the \"Birthplace of ROTC\" (Reserve Officers\\' Training Corps).']], ['Kogakuin University', ['Kogakuin University (工学院大学 , Kōgakuin daigaku ) is a private university in Shinjuku, Tokyo, Japan.', ' The predecessor of the school initially named \"Koshu Gakko\" which is one of the oldest private engineering schools in Japan, was founded in 1887 by educator and politician, The President of Tokyo Imperial University\\'s President Koki Watanabe and other professors of Tokyo Imperial University.']], [\"2012–13 Mercer Bears men's basketball team\", [\"The 2012–13 Mercer Bears men's basketball team represented Mercer University during the 2012–13 NCAA Division I men's basketball season.\", \" The Bears, led by fifth year head coach Bob Hoffman, played their home games at Hawkins Arena on the university's Macon, Georgia campus and were members of the Atlantic Sun Conference.\", ' They finished the season 24–12, 14–4 in A-Sun play to win the regular season conference championship.', ' They advanced to the championship game of the Atlantic Sun Tournament where they lost to Florida Gulf Coast.', ' As a regular season conference champions who failed to win their conference tournament, they received an automatic bid to the 2013 NIT where they defeated Tennessee in the first round before losing in the second round to BYU.']], ['Universitas Nasional', ['Universitas Nasional (UNAS or Nasional University) is the oldest private university in Jakarta and the second oldest in Indonesia.', ' It was founded by several Indonesian scholars on October 15, 1949, as Akademi Nasional.']], ['Mercer University', ['Mercer University is the oldest private university in Georgia with its main campus in Macon, Georgia, United States.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.518\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae08a0455429924de1b70fc', 'answer': 'Enigma', 'question': 'Which German project recorded a song that featured vocals by a duo from Silverdale, England?', 'supporting_facts': [['Amen (Enigma song)', 0], ['Aquilo (band)', 0]], 'context': [['The NeverEnding Story (song)', ['\"The NeverEnding Story\" is the title song from the English version of the 1984 film \"The NeverEnding Story\".', ' It was performed by Limahl.', ' Limahl released two versions of the song, one in English and one in French.', ' The English version featured vocals by Beth Anderson, and the French version featured vocals by Ann Calvert.', ' It was a success in many countries, reaching No. 1 in Norway and Sweden, No. 2 in Austria, Germany and Italy, No. 4 in the UK, No. 6 in Australia and No. 6 in the US Billboard Adult Contemporary chart.']], ['Troum', ['Troum is a German project of drone music, ambient music, noise music, and experimental music.', ' It was founded in the late 1990s by Stefan Knappe (a.k.a. Baraka[H]) and Martin Gitschel (a.k.a. Glit[S]ch).', ' It is sometimes considered to be the follow-up project to Maeror Tri.', ' Stefan Knappe is also the founder and owner of Drone Records.']], ['Pierre Célestin Munyanshongore', ['Pierre Célestin Munyanshongore (born 1942 in Butare province and died in 2011) was an ethnic Hutu engineer in Rwanda.', ' He attended university in Germany and graduated with a Mechanical Engineering degree in the 1960s.', ' He was the director of a German project until 1994.', ' At the end of the 1994 genocide Munyanshongore was arrested and released after eight years in prison.', ' Before his death on November 27, 2011, he ran an education development project based in the Eastern province of Rwanda.']], ['Run the World', ['\"Run the World\" is a song recorded by American entertainer Jennifer Lopez for her seventh studio album \"Love?', '\" (2011).', ' Written and produced by Terius \"The-Dream\" Nash and C. \"Tricky\" Stewart, \"Run the World\" was one of several songs recorded with the duo following Lopez\\'s move from Epic Records to Island Records.', ' The track originally featured vocals from The-Dream and a rap verse from American rapper Rick Ross however, the rap verse was removed during mastering process and The-Dream was credited with background vocals instead.']], ['Deutsche Wirtschaftsbetriebe', ['Deutsche Wirtschaftsbetriebe (German: \"for \\'German Economic Enterprises\"\\' ) abbreviated DWB, was a Nazi German project launched in World War II by the Allgemeine SS to profit from the use of forced and compulsory labour extracted from the Nazi concentration camp inmates.']], ['Red Nation', ['\"Red Nation\" is a song by American rapper and West Coast hip hop artist Game featuring vocals from rapper Lil Wayne, from his anticipated fourth studio album \"The R.E.D. Album\".', ' Released as the album\\'s lead single on April 12, 2011, the song was written by Game and Lil Wayne, and it was produced by Miami-based production duo Cool & Dre, noted for producing three of Game\\'s singles including the 2005 smash hit single \"Hate It or Love It\" which featured vocals from former fellow G-Unit member rapper 50 Cent.', ' The single also marks the second collaboration between Game and Lil Wayne, their first collaboration being on Game\\'s 2008 international hit single \"My Life\" from his third studio album \"LAX\" (2008), which featured vocals from Lil Wayne during the song\\'s chorus.', ' The song features a music sample of the nightclub hit \"Kernkraft 400\" (2000) (German for \"Nuclear Energy 400\") performed by German techno and electro band Zombie Nation from their debut album \"Leichenschmaus\" (1999).']], ['Aquilo (band)', ['Aquilo is an alternative musical duo from Silverdale, Lancashire, England, consisting of Tom Higham and Ben Fletcher.', ' They began gaining recognition in 2013 for their singles such as \"Calling Me\" and \"You There\", in addition to their five EPs.']], ['Endless Summer (Oceana song)', ['\"Endless Summer\" is a song by German singer Oceana Mahlmann, from her second album \"My House\" (2012), serving as the lead single.', ' It was the official UEFA Euro 2012 theme song.', ' The song uses a sample of the electro track \"Blaue Moschee\" by German project Die Vögel.']], ['Amen (Enigma song)', ['\"Amen\" is a song by German musical project Enigma, featuring vocals by English dream-pop duo Aquilo.', ' It was released as the second single from Enigma\\'s eighth studio album, \"The Fall of a Rebel Angel\" on November 18, 2016.']], ['Borghild Project', ['The Borghild Project was a hoax purported to be evidence for a German project during World War II aimed at combating the spread of syphilis among Nazi troops by supplying soldiers with sex dolls.', ' Adolf Hitler supposedly approved the project to distribute inflatable sex dolls to his soldiers, which could be transported in their backpacks in order to give them an option to avoid places of prostitution in Paris.', ' After years of being considered a real project, the lack of evidence supporting its existence led to it being deemed a hoax in the early 2000s, for various reasons.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.518\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab484415542990594ba9c44', 'answer': 'Discovery Zone', 'question': \"Which facility was founded in Missouri, Discovery Zone or Valentino's?\", 'supporting_facts': [['Discovery Zone', 0], ['Discovery Zone', 3], [\"Valentino's\", 0]], 'context': [['Leaps and Bounds (playplace)', [\"Leaps and Bounds was a chain of indoor play-places that was started by McDonald's in 1991.\", ' It was merged with competitor Discovery Zone in 1995.']], ['Webb Mountain Discovery Zone', ['Webb Mountain Discovery Zone is a park in Monroe, Connecticut, United States.', ' Its mission is to provide excellence in outdoor education and to promote exploration and learning in a fun, hands-on environment as a prototype for the Federal No Child Left Inside initiative.', ' The park covers 170 acre and has 3 loop trails, each with marked interpretive signs.', ' There is also an outdoor classroom for schools and groups, and a scavenger hunt scorecard.', ' The park provides additional educational resources for teachers, as well as programs for school groups, daycare centers, and scouts.', ' The park is owned by the Town of Monroe and administered by the Friends of Webb Mountain.']], ['Herald House', ['Herald House or Herald Publishing House is the publishing division of the Community of Christ in Independence, Missouri.', ' It publishes books, periodicals and other materials at the direction of the First Presidency.', ' Its history dates to the publication of a church periodical called the \"True Latter Day Saints\\' Herald\" in Cincinnati, Ohio in 1860.', ' The first church-owned press was located in Plano, Illinois and a much larger facility was opened in Lamoni, Iowa in 1881.', ' The publishing plant in Lamoni was destroyed by fire in 1907.', ' A replacement facility was built shortly thereafter.', ' When the church headquarters moved to Independence, Missouri in 1921, the Herald House was relocated to a facility that had previously been used by an artillery battalion of the Missouri National Guard.', ' In 1965, a modern publishing facility was built for Herald House on Noland Road in Independence.', ' That facility was closed in 1999 and printing has been outsourced since that date.', ' The publishing offices were moved to The Temple and The Auditorium in Independence, Missouri.']], ['Great Falls Tectonic Zone', ['The Great Falls Tectonic Zone is a major intracontinental shear zone between the Hearne craton and Wyoming craton basement rock of the Archean Eon which form part of the North American continent.', ' The zone is an area about 100 miles (150 km) wide extending from the southwestern Idaho-Montana border across Montana to the northwestern Montana-Saskatchewan-North Dakota border.', ' It is named for the Great Falls of the Missouri River, a major geologic feature of the area.', ' The central and western portions of the zone are believed to be about 1.1 to 3.3 billion years old.', ' The central part of the zone lacks Archean rock, however, leading at least one group of scientists to speculate that it was formed very late in the Paleoproterozoic Era.']], ['University of North Texas Discovery Park', ['The University of North Texas Discovery Park Campus, formerly Research Park, is a satellite research facility of the University of North Texas.', ' Discovery Park is located in Denton, Texas, north of the main campus, on U.S. Highway 77.', ' In January 2004, the 550000 sqft facility, formerly occupied by Texas Instruments, opened to students from the UNT College of Engineering.', ' In 2008, the newly formed College of Information joined the Discovery Park campus.', ' The facility houses offices and labs for the Departments of Engineering Technology, Computer Science and Engineering, Materials Science and Engineering, Electrical Engineering, Mechanical and Energy Engineering, Library and Information Science and Learning Technologies.', ' The Center for Technology Development and Transfer (CTDT) began operations from Discovery Park in 2006.']], ['Discovery Zone', ['Discovery Zone (DZ) was a chain of entertainment facilities featuring games and elaborate indoor mazes designed for young children, including roller slides, climbing play structures and ball pits.', ' It also featured arcade games.', ' The chain was founded by Ronald Matsch, Jim Jorgensen and Dr. David Schoenstadt in 1989.', ' The first location was opened in Kansas City, Missouri in October 1989.', ' An early investor and vocal supporter of the company was tennis player Billie Jean King.']], [\"Valentino's\", [\"Valentino's is a regional Italian restaurant chain based in Lincoln, Nebraska.\", \" Valentino's was founded by Val and Zena Weiler in 1957.\", ' The restaurant was purchased by two Lincoln families in 1971 and began franchising additional locations.', ' The first carry-out store opened in 1990, and many of the full-scale restaurants converted to the buffet concept in the early-2000s.']], ['Valentino SpA', ['Valentino SpA is a clothing company founded in 1960 by Valentino Garavani.', ' It is a part of Valentino Fashion Group, which in turn is owned by the State of Qatar through Mayhoola for Investments S.P.C.', ' Since October 2008, the creative director is Pier Paolo Piccioli.', \" Alessandra Facchinetti was Valentino's creative designer from 2007 to 2008.\", ' Valentino is headquartered in Milan,while the creative direction is in Rome.']], ['Aliso Canyon Oil Field', ['The Aliso Canyon Oil Field (also Aliso Canyon Natural Gas Storage Field, Aliso Canyon Underground Storage Facility) is an oil field and natural gas storage facility in the Santa Susana Mountains in Los Angeles County, California, north of the Porter Ranch neighborhood of the City of Los Angeles.', ' Discovered in 1938 and quickly developed afterward, the field peaked as an oil producer in the 1950s, but has remained active since its discovery.', ' One of its depleted oil and gas producing formations, the Sesnon-Frew zone, was converted into a gas storage reservoir in 1973 by the Southern California Gas Company, the gas utility servicing the southern half of California.', ' This reservoir is the second-largest natural gas storage site in the western United States, with a capacity of over 86 billion cubic feet of natural gas.', ' Currently it is one of four gas storage facilities owned by Southern California Gas, the others being the La Goleta Gas Field west of Santa Barbara, Honor Rancho near Newhall, and Playa del Rey.']], ['Missouri Theatre (Columbia, Missouri)', ['The Missouri Theatre, is a concert and entertainment venue in downtown Columbia, Missouri, occupying most of a city block between 9th street between Locust and Elm Streets.', ' It was designed after the Opéra Garnier by the Boller Brothers, built in 1928, and is on the National Register of Historic Places.', \" It is Columbia's only surviving pre-Depression movie palace and vaudeville stage.\", ' In 2011, the University of Missouri began a three-year lease of the facility.', ' The Missouri Theatre is the resident home of the Missouri Symphony Orchestra, and is also frequently used by University of Missouri and civic groups.', ' As of July 1, 2014, The University of Missouri took over ownership of the Missouri Theatre.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.519\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a80071f5542992bc0c4a684', 'answer': 'Crystal Dynamics', 'question': 'Alice David is the voice of Lara Croft in a video game developed by which company ?', 'supporting_facts': [['Alice David', 1], ['Tomb Raider (2013 video game)', 0]], 'context': [['List of Tomb Raider media', ['Tomb Raider is a media franchise consisting of action-adventure games, comic books, novels, theme park rides, and movies, centring on the adventures of the female fictional British archaeologist Lara Croft.', ' Since the release of the original \"Tomb Raider\" in 1996, the series developed into a lucrative franchise of related media, and Lara went on to become a major icon of the video game industry.', ' The \"Guinness Book of World Records\" has recognised Lara Croft as the \"Most Successful Human Videogame Heroine\" in 2006.', ' Six games in the series were developed by Core Design, and the latest four by Crystal Dynamics.', ' All the games were first published by Eidos Interactive, now Eidos officially became part of Square Enix on 22 April 2009, meaning Square Enix owns the rights to the \"Tomb Raider\" trademark and characters of the franchise.', ' To date two movies, \"\" and \"\", have been produced starring American actress Angelina Jolie as Lara Croft.', ' A reboot was announced in 2016.']], ['Tomb Raider', ['Tomb Raider, also known as Lara Croft: Tomb Raider between 2001 and 2007, is a media franchise that originated with an action-adventure video game series created by British gaming company Core Design.', ' Formerly owned by Eidos Interactive, then by Square Enix after their acquisition of Eidos in 2009, the franchise focuses on a fictional English archaeologist Lara Croft, who travels around the world searching for lost artifacts and infiltrating dangerous tombs and ruins.', ' The gameplay generally focuses around action-adventure exploration of environments, solving puzzles, navigating hostile environments filled with traps, and fighting numerous enemies.', ' Additional media has grown up around the theme in the form of film adaptations, comics and novels.']], ['Lara Croft Go', ['Lara Croft Go is a 2015 turn-based puzzle video game in the \"Tomb Raider\" series.', ' The player moves Lara Croft as a puzzle piece through a board game while avoiding obstacles and manipulating the environment.', ' The developers distilled major series motifs, such as boulder chases and reaction-based gameplay, to suit \"Lara Croft Go\"\\'s time-independent gameplay.', ' Square Enix Montreal developed the game as a spiritual successor to its 2014 \"Hitman Go\", based on another Square Enix franchise.', ' The company released \"Lara Croft Go\" in August 2015 for Android, iOS, Windows, and Windows Phone devices.', ' A version for PlayStation 4 and PlayStation Vita was unveiled in November 2016.']], ['Tomb Raider III', ['Tomb Raider III: Adventures of Lara Croft, or simply Tomb Raider III, is an action-adventure video game developed by Core Design and published by Eidos Interactive.', ' It was originally released for the PlayStation and Microsoft Windows platforms in 1998.', ' \"Tomb Raider III\" is the third title in the \"Tomb Raider\" video game series and a sequel to \"Tomb Raider II\".', ' The story of the game follows archaeologist-adventurer Lara Croft as she embarks upon a quest to recover four pieces of a meteorite that are scattered across the world.', ' To progress through the game, the player must explore five locations (India, South Pacific, London, Nevada, and Antarctica) and complete a series of levels that involve solving puzzles, jumping over obstacles, and defeating enemies.']], ['Lara Croft and the Temple of Osiris', ['Lara Croft and the Temple of Osiris is an action-adventure game developed by Crystal Dynamics and published by Square Enix for Microsoft Windows, PlayStation 4 and Xbox One.', ' It is the sequel to the 2010 video game \"Lara Croft and the Guardian of Light\", and the second instalment in \"Lara Croft\" spin-off series of the \"Tomb Raider\" franchise.', ' The video game was announced at E3 2014 on 9 June.', ' \"Temple of Osiris\" stars \"Tomb Raider\" mainstay protagonist Lara Croft, voiced by English actress Keeley Hawes.']], ['Lara Croft: Tomb Raider', ['Lara Croft: Tomb Raider (also known as simply Tomb Raider) is a 2001 action-adventure film based on the popular \"Tomb Raider\" video game series featuring the character Lara Croft portrayed by Angelina Jolie.', ' An international co-production between the United States, the United Kingdom, Japan, and Germany, it was directed by Simon West and revolves around Lara Croft trying to obtain ancient artifacts from the enemy, the Illuminati.']], ['Lara Croft and the Guardian of Light', ['Lara Croft and the Guardian of Light is an action-adventure game developed by Crystal Dynamics and published by Square Enix for Microsoft Windows, PlayStation 3, Xbox 360, Android and, iOS through digital distribution.', ' It is part of the \"Tomb Raider\" series, but unlike previous games, the game does not carry the \"Tomb Raider\" brand and has a heavy emphasis on cooperative gameplay.', ' In multiplayer, players take the role as either Lara Croft or a 2,000-year-old Mayan warrior named Totec.', ' They must work together in order to stop the evil spirit Xolotl and retrieve the Mirror of Smoke.', ' A single-player campaign mode is available that does not include the non-playable character AI following or helping Lara.']], ['Tomb Raider (2013 video game)', ['Tomb Raider is an action-adventure video game developed by Crystal Dynamics and published by Square Enix.', ' \"Tomb Raider\" is the tenth title in the \"Tomb Raider\" franchise, and operates as a reboot that reconstructs the origins of Lara Croft.', ' \"Tomb Raider\" was released on 5 March 2013 for Microsoft Windows, PlayStation 3 and Xbox 360, and on 23 January 2014 for OS X, and on 27 April 2016 for Linux.']], ['Lara Croft', ['Lara Croft is a fictional character and the main protagonist of the Square Enix (previously Eidos Interactive) video game franchise \"Tomb Raider\".', ' She is presented as a highly intelligent, athletic, and beautiful English archaeologist-adventurer who ventures into ancient, hazardous tombs and ruins around the world.', ' Created by a team at UK developer Core Design that included Toby Gard, the character first appeared in the 1996 video game \"Tomb Raider\".', ' She has also appeared in video game sequels, printed adaptations, a series of animated short films, feature films (portrayed by Angelina Jolie, later by Alicia Vikander), and merchandise related to the series.', ' Official promotion of the character includes a brand of apparel and accessories, action figures, and model portrayals.', ' Croft has also been licensed for third-party promotion, including television and print advertisements, music-related appearances, and as a spokesmodel.', ' As of June 2016, Lara Croft has been featured on over 1,100 magazine covers surpassing any supermodel.']], ['Alice David', ['Alice David (born 22 March 1987) is a French actress, known for the television series \"Bref\" (2011) and the film \"Babysitting\" (2014).', ' She is the voice of the French dub of Lara Croft in the video game \"Tomb Raider\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.520\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a811667554299260e20a23d', 'answer': 'singer, songwriter', 'question': 'What occupations do both Ian Hunter and Rob Thomas have?', 'supporting_facts': [['Ian Hunter (singer)', 0], ['Rob Thomas (musician)', 0]], 'context': [['The Ballad of Ian Hunter and Mott the Hoople', [\"Shades of Ian Hunter: The Ballad of Ian Hunter and Mott the Hoople is a compilation album by Ian Hunter, consisting of tracks by Hunter's previous band Mott the Hoople, and solo Hunter tracks as well.\", ' It was released in 1979 as a double-LP.']], ['Once Bitten, Twice Shy', ['\"Once Bitten, Twice Shy\" is a 1975 song written and recorded by Ian Hunter, from his debut solo album \"Ian Hunter\", which reached No. 14 in the UK Singles Chart.']], ['Ian Hunter (album)', ['Ian Hunter is the first solo album by Ian Hunter, recorded following his departure from Mott the Hoople.', ' Released in 1975, it is also the first of many solo albums on which he collaborates with Mick Ronson.', ' The bassist, Geoff Appleby, was from Hull like Mick Ronson and they had played together in The Rats in the late 1960s.']], ['Ian Hunter (singer)', ['Ian Hunter Patterson (born 3 June 1939), known as Ian Hunter, is a British singer-songwriter who is best known as the lead singer of the English rock band Mott the Hoople, from its inception in 1969 to its dissolution in 1974, and at the time of its 2009 and 2013 reunions.', ' Hunter was a musician and songwriter before joining Mott the Hoople, and continued in this vein after he left the band.', ' He embarked on a solo career despite ill health and disillusionment with commercial success, and often worked in collaboration with Mick Ronson, David Bowie\\'s sideman and arranger from the \"Ziggy Stardust and the Spiders from Mars\" period.']], ['Windsor Festival', ['The Windsor Festival was founded in 1969 with Yehudi Menuhin and Ian Hunter as Artistic Directors and Laurence West as Executive Chairman.', \" The original idea for the Festival was put forward by Ian Hunter to the Dean of Windsor in 1968, building on the participation of the Menuhin Festival Orchestra with Yehudi Menuhin using St George's Chapel, the State Apartments of Windsor Castle and the Theatre Royal.\", ' The Dean formed the Windsor Festival Society, which then moved to plan the first festival.']], ['Veronica Mars', ['Veronica Mars is an American teen noir mystery drama television series created by screenwriter Rob Thomas.', ' The series is set in the fictional town of Neptune, California, and stars Kristen Bell as the eponymous character.', \" The series premiered on September 22, 2004, during television network UPN's final two years, and ended on May 22, 2007, after a season on UPN's successor, The CW, airing for three seasons total.\", ' \"Veronica Mars\" was produced by Warner Bros.', ' Television, Silver Pictures Television, Stu Segall Productions, and Rob Thomas Productions.', ' Joel Silver and Rob Thomas were executive producers for the entire run of the series, while Diane Ruggiero was promoted in the third season.']], ['Veronica Mars (character)', ['Veronica Mars is the fictional protagonist, occasional narrator (through voiceovers), and antiheroine of the American television series \"Veronica Mars\", which aired on UPN from 2004 to 2006 and on The CW from 2006 to 2007.', ' The character was portrayed by Kristen Bell through the duration of the series.', \" Following the show's cancellation, Bell reprised the role in the 2014 film continuation.\", ' The character, created by Rob Thomas, was originally male and the protagonist of his unproduced novel \"Untitled Rob Thomas Teen Detective Novel\", which eventually became the basis of the series.', \" After the work's transition from novel to television series, Thomas changed the character's gender from male to female as he believed a noir piece told from a female point of view would be more interesting.\"]], ['Kjetil Bjerkestrand', ['Kjetil Bjerkestrand (born 18 May 1955 in Kristiansund, Norway) is a Norwegian musician (keyboards), composer, arranger and record producer, known as music arranger for artists like Ray Charles, Dee Dee Bridgewater, Keith Emerson, Ian Hunter, Jon Lord, Ute Lemper and a-ha.', ' As a musician, he has participated in recordings with a-ha, Ray Charles, Ute Lemper, Ian Hunter, Dance with a Stranger, DumDum Boys, Jonas Fjeld Band, Marius Müller, TNT, Arve Tellefsen, Bobbysocks, Bjørn Eidsvåg, Carola Häggkvist and Dee Dee Bridgewater.']], ['Rob Thomas (musician)', ['Robert Kelly Thomas (born February 14, 1972) is an American singer, songwriter, record producer and multi-instrumentalist, best known as the lead singer of Alternative band Matchbox 20.', ' Thomas also records and performs as a solo artist with \"Lonely No More\" released in 2005 becoming his biggest solo chart success.', ' Thomas earned three Grammy Awards for co-writing and singing on the three-time Grammy Award Winning 1999 Summer smash hit, \"Smooth\" by Santana, off the fifteen-time Platinum album \"Supernatural\".']], [\"Short Back 'n' Sides\", [\"Short Back 'n' Sides is the fifth solo album of Ian Hunter.\", \" Unsure of which direction he should take, Ian Hunter finally decided to collaborate with Mick Jones, who gave Hunter's songs a tougher and heavier touch.\", ' Fellow Clash member Topper Headon as well as Mick Ronson, Todd Rundgren and Ellen Foley also appeared on this album.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.520\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a81c7d15542990a1d231ea9', 'answer': 'cleaning, catering and security', 'question': 'Which services did Rock Nominees Ltd and  ISS A/S (Integrated Service Solutions) have in common?', 'supporting_facts': [['Rock (Nominees) Ltd v RCO Holdings Ltd', 4], ['ISS A/S', 1]], 'context': [['FTS2000', ['Federal Telecommunications System 2000 (FTS2000) is a long distance telecommunications service for the United States federal government, including services such as switched voice service for voice or data up to 4.8 kbit/s, switched data at 56 kbit/s and 64 kbit/s, switched digital integrated service for voice, data, image, and video up to 1.544 Mbit/s, packet switched service for data in packet form, video transmission for both compressed and wideband video, and dedicated point-to-point private line for voice and data.']], ['Advanced Digital Broadcast', ['ADB is a company which provides and integrates software, system and service solutions to service providers and Pay-TV operator delivering connected services for connected lives.']], ['Integrated Service Provider', ['An Integrated Service Provider (ISP) is a for-hire firm that performs a variety of logistics service activities such as warehousing, transportation, and other functional activities that constitute a total service package.', \" In addition, other categories of spend may fall under the ISP's scope such as maintenance, repair, and operations (MRO) services.\", ' Firms that provide such services typically have a good understanding of their customers needs and are responsible for executing services in accordance with contract documents.', ' Normally the scope of work (SOW) and the ISP contract are finalized only after an extensive due diligence period.']], ['ISS A/S', ['ISS A/S (Integrated Service Solutions) is a Facility Services company founded in Copenhagen, Denmark in 1901.', ' ISS services include: cleaning services, support services, property services, catering services, security services and facility management services.', ' The ISS Group’s revenue amounted to DKK 79.1 billion in 2016 and ISS has nearly 500,000 employees and activities in approximately 75 countries across Europe, Asia, North America, Latin America and Pacific.', \" More than half of ISS' employees are based in emerging markets.\"]], ['Tang Jun (executive)', ['Tang Jun () is the President of Gaotime Information Co. Ltd, a consulting service solutions provider for both financial institutions and large companies within China.']], ['B2X GmbH', ['B2X GmbH is a business process outsourcing company.', ' B2X gives customer service solutions for manufacturers of smartphones and other electronic devices, insurance providers, mobile network operators and retailers.', ' The services are based on a technology platform called SMARTCARE Technology.', ' Although its headquarters are located in Munich, Germany, the company works in more than 130 countries through a network of over 400 service partners and 2,000 service locations.']], ['Downsview Nominees Ltd v First City Corp Ltd', ['Downsview Nominees Ltd v First City Corp Ltd [1992] UKPC 34 is a New Zealand insolvency law case decided by the Judicial Committee of the Privy Council concerning the nature and extent of the liability of a mortgagee, or a receiver and manager, to a mortgagor or a subsequent debenture holder for his actions.']], ['Rock (Nominees) Ltd v RCO Holdings Ltd', ['Rock Nominees Ltd was part of the business empire of Lord Ashcroft, a Tory peer who has been criticised for offshore tax avoidance.', ' It is a company which holds shares on behalf of other companies.', ' It had 201,300 shares for Gambier Holdings Inc. (a British Virgin Islands company) and 65,000 shares for Kiwi Ltd. (a Belize company) invested in RCO (Holdings) plc.', ' Its stake made up 2.48%.', ' RCO itself was in the cleaning, catering and security porterage business.', ' In 2000 a company called ISS (UK) Ltd took over RCO, acquiring 96.4% of the shares.', \" It made one of RCO's subsidiaries transfer its shares to one of ISS's subsidiaries for £30,117,784.\", \" Rock Nominee's filed for a petition of unfair prejudice on the grounds that this was a transaction at an undervalue.\", ' It did not reflect the value to the purchaser of the synergies arising from the sale or the value of avoiding risk from a sale on the open market.']], ['Daulia Ltd v Four Millbank Nominees Ltd', ['Daulia Ltd v Four Millbank Nominees Ltd [1977] EWCA Civ 5 is an English contract law case, concerning unilateral contracts, and when embarking on the performance of an act for which an offer is open, at what point the offer may be withdrawn.', ' In particular, Goff LJ observed that there would be a duty to not prevent full performance of terms in a unilateral offer, once performance had begun.']], ['Integrated Broadband Services', ['Integrated Broadband Services (IBBS) provides fully integrated, cloud-based data and voice solutions to broadband providers in the United States, Canada, Latin America, the Caribbean, and Brazil.', ' Serving over 250 broadband providers that support more than 1.5 million modems worldwide, IBBS provides both residential broadband services and commercial service solutions to broadband operators.', ' IBBS also provides services in provisioning, diagnostics, engineering, development, network management, VoIP and technical support services.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.521\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a716ec85542994082a3e82d', 'answer': \"You're Next\", 'question': 'Which movie was filmed first \"The Guest\" or \"You\\'re Next\"?', 'supporting_facts': [['The Guest (film)', 0], [\"You're Next\", 0]], 'context': [['K. Ravindran Nair', ['K. Ravindran Nair was born in a rich family, dealing with cashew production and exports.', ' His passion for literature and arts brought him into Malayalam cinema and in 1967, he established \"General Pictures\" under the banner of which he produced his first movie, Anweshichu Kandethiyilla, directed by P. Bhaskaran.', ' This was followed by two more films the next year, Kattukurangu and Lakshaprabhu, both directed by Bhaskaran.', ' Ravi, as he is generally known, was silent for the next few years till he came out with his next film, Achani, an A. Vincent movie, in 1973, which earned him the moniker, \"Achani Ravi\".', ' The film was reported to be a commercial success like his earlier films and Ravi is known to have contributed the returns from the movie for building a Public Library in Kollam, of which he is a founder member and honorary secretary.']], ['The White Hell of Pitz Palu', ['The White Hell of Pitz Palu (German: \"Die weisse Hölle vom Piz Palü\" ) is a 1929 German silent mountain film co-directed by Arnold Fanck and Georg Wilhelm Pabst and starring Leni Riefenstahl, Gustav Diessl, Ernst Petersen, and World War I flying ace Ernst Udet.', ' Written by Arnold Fanck and Ladislaus Vajda, the film is about a man who loses his wife in an avalanche while climbing the Piz Palü mountain, and spends the next few years searching the mountain alone for her body.', ' Four years later he meets a young couple who agree to accompany him on his next climb.', ' \"The White Hell of Pitz Palu\" was filmed on location in the Bernina Range in Graubünden, Switzerland.', ' The 1929 theatrical release starred Kurt Gerron, who was Jewish, as a night club guest.', ' The film was edited to remove scenes featuring Gerron, and it was rereleased as a 90-minute German-language sound film in 1935.', ' It was remade in 1950.']], ['Asturian cinema', ['The Asturian Cinema (or \"Asturian National Cinema\") in Asturias, Spain, began in 1905 with the production of the first Asturian fiction film known as \"El robo de fruta\" (The fruit robbery).', ' It was filmed by the Asturian film maker Javier Sánchez Manteola.', ' This movie was filmed in Gijón (Asturias) and premiered in that city in the same year.', ' It was shown in the old movie theater known as Salón Luminoso once located at Begoña Walk.', ' In commemoration of this even the Asturian Film Festival was established in the year 2005.']], ['The Guest (film)', ['The Guest is a 2014 American action horror-thriller film directed and edited by Adam Wingard and written by Simon Barrett, both of whom previously collaborated on a previous film, \"You\\'re Next\" (2011).', ' Starring Dan Stevens, Maika Monroe, Leland Orser, Sheila Kelley, Brendan Meyer, and Lance Reddick, the plot follows a soldier named \"David\" unexpectedly visiting the Peterson family, introducing himself as a friend of their son who had died during the Afghanistan war.', ' After the man is welcomed into their home for a couple of days, a series of deaths begin to occur around his presence, and their daughter Anna begins to have suspicions of David being connected to the deaths.']], ['Stephen Nicholas (actor)', ['Stephen Nicholas (born 23 August 1978) also known as Stephen Charles Nicholas is an actor and presenter from Doncaster, South Yorkshire, England.', \" Stephen currently lives in Sheffield, his first role was on Sky One's Dream Team, where he played Scott Ward.\", ' From there, he filmed the first in the trilogy Goal!', ' (In which he played a Newcastle United Reserves player).', ' Following this, he moved to Los Angeles, where he played Smith in the feature film Futbaal: The Price of Dreams.', ' Stephen then returned to the UK to make a Bollywood film called Dhana Dhana Goal with John Abraham.', ' Stephen then experienced his first opportunity in reality TV with the show Premier League All Stars for Sky One, as well as playing a footballer, he was on-hand to present celebrity gossip and pitch side reports.', ' He then appeared in Celebrity Most Haunted and Date the Enemy.', ' From there he then went on to star in Goal 3 where he not only acted in the film he also became the football choreographer and choreographed all the football scenes in the film.', ' Nicholas then starred in the film Damned United where he played Welsh international Alan Durban, the film was filmed in Chesterfield and Leeds and was directed by Oscar winner Tom Hooper and also starred Oscar nominated Michael Sheen.', \" Stephens next production was the feature film called 'No Way Back Now'about the notorious Manchester district of Moss Side, where Stephen played the lead actor Stuart Gavin,The feature is roughly based on the notorious Gooch gang that terrorised Manchester throughout the years.\", ' The next move for Stephen was pantomime where he was part of the production Aladdin over the Christmas period of 2015 in Doncaster playing Abanaza the main villain which he did until January 7, 2016!', '.', \" He has recently been cast in the up-and-coming Feature Film 'Whiteblade' where he will play Thurstan the head Warlord Whiteblade is currently in production and Stephen is shooting his scenes in August 2016.\", \" In September 2016 Stephen will be presenting the Sky TV show 'Britz go Bollywood' the show consists of a group of Celebrities being dressed by The best Indian designers, Stephen is the main presenter of the show which will be screened live September 2, 2016.\"]], [\"Live from Daryl's House\", [\"Live from Daryl's House (simply known as Daryl's House, and often abbreviated as LFDH) is an online series that was first created in fall 2007.\", ' The show features singer-songwriter Daryl Hall performing with his band and various guest artists at his home in Millerton, New York.', ' The show provides a performance space that is an alternative to live concerts and studio sessions for popular artists.', ' This allows the artists to \"…have fun and [be] creatively spontaneous\".', ' The majority of shows include a segment in which Hall and the guest artist prepare food from different cuisines for everyone to eat.', ' The food comes from various local restaurants and the chefs of those establishments walk Hall and guest through the preparation of the food.', ' \"Live From Daryl\\'s House\" expanded to broadcast TV but remained unchanged.', ' Hall was quoted by Billboard.com as saying \"it\\'s an Internet show that is being shown on television, so I\\'m not adapting the show at all in any way to be a \\'TV\\' show.\"', ' The show debuted in 95 markets on September 24, 2011, with back-to-back half-hour episodes featuring Train (Episode 33) and Fitz & the Tantrums (Episode 35).', ' Starting with the 66th episode of \"Live From Daryl\\'s House\", the shows are filmed at Hall\\'s club, Daryl\\'s House, in Pawling, New York.']], [\"You're Next\", [\"You're Next is a 2011 American slasher film directed by Adam Wingard, written by Simon Barrett and starring Sharni Vinson, Nicholas Tucci, Wendy Glenn, A. J. Bowen and Joe Swanberg.\", ' The plot concerns a family under attack by a group of masked assailants during their wedding anniversary getaway.']], ['Rose Marie (1954 film)', ['Rose Marie is a 1954 musical adaptation of the 1924 operetta of the same name, the third to be filmed by Metro-Goldwyn-Mayer, following a 1928 silent movie and the best-known of the three, the 1936 Jeanette MacDonald/Nelson Eddy version.', ' It is directed by Mervyn LeRoy and stars Ann Blyth, Howard Keel and Fernando Lamas.', ' This version is filmed in the Canadian Rockies in CinemaScope.', \" It was MGM's first US produced film in the new widescreen medium (having been preceded by the British made Knights of the Round Table) and the first movie musical of any studio to be released in this format.\"]], ['The Pin Up Girls', ['The Pin Up Girls are a girl group and dance troupe, founded by New York City native Vixen Romeo in 2005, which began as a burlesque-style performance group based in Los Angeles.', \" Performing at Hollywood's most notorious venues such as The Viper Room Key Club and Roxy the girls quickly gained local attention with their girl-on-girl themed, tribal fusion belly dance, burlesque and hip hop routines.\", ' Between 2006-2008 The Pin Up Girls started to become poster girls for the lesbian scene with performances for Curve (magazine), a guest appearance on LOGO network\\'s reality series \"Curl Girls\", a web series segment on AfterEllen, a performance for the LGBT community hosted by Jane Lynch, and performances in Margaret Cho\\'s Sensuous Woman Show.', ' In 2008 The Pin Up Girls first recorded single \"There She Goes...She\\'s Real Fly\" was picked up to be played on Showtime\\'s hit lesbian series \"The L Word\".', ' In 2009 The Pin Up Girls music video, \"There She Goes...She\\'s Real Fly\" premiered on Logo (TV channel) (an MTV network), on New Now Next Pop Lab.', ' The Pin Up Girls\\' \"Girl Candy,\" filmed in N.Y. and L.A., was released in 2011.', ' The Pin Up Girls\\' \"Pretty Things\", featuring actress Elaine Hendrix, was filmed in L.A. by Director Joe LaRue in 2012 and was released in June 2012.']], ['Live from Abbey Road', ['Live from Abbey Road is a 12-part, one-hour performance series/documentary that began filming its first season during 2006 at Abbey Road Studios in London.', ' Season 2 was filmed between 2007 and 2008, season 3 was filmed in 2009 and Season 4 was filmed in 2011.', ' The series features a total of 128 musical artists to date (about 32 per Season) -- usually two or three per show, performing up to five songs per session.', ' The sessions are recorded without a live audience.', ' Filmed in High-Definition with the occasional use of 35 mm lenses, the producers have sought to record performances which \"look like a movie and sound like a record\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.522\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ade25ed5542997c77aded70', 'answer': 'the Cold War (1947–91)', 'question': 'During what war were the Russia-United Kingdom relations in a state of rivalry after the abdication of Emperor Nicholas II? ', 'supporting_facts': [['Russia–United Kingdom relations', 3], ['Russia–United Kingdom relations', 4], ['Russian Revolution', 1]], 'context': [['Grand Duke Vladimir Alexandrovich of Russia', ['Grand Duke Vladimir Alexandrovich of Russia (\"Влади́мир Александрович\") ) (22 April 1847 – 17 February 1909) was a son of Emperor Alexander II of Russia, a brother of Emperor Alexander III of Russia and the senior Grand Duke of the House of Romanov during the reign of his nephew, Emperor Nicholas II.']], ['Grand Duke Sergei Alexandrovich of Russia', ['Grand Duke Sergei Alexandrovich of Russia (\"Сергей Александрович\"; May 11, 1857 – February 17, 1905) was the fifth son and seventh child of Emperor Alexander II of Russia.', \" He was an influential figure during the reigns of his brother Emperor Alexander III of Russia and his nephew Emperor Nicholas II, who was also his brother in law through Sergei's marriage to Elizabeth the sister of Tsarina Alexandra.\"]], ['Charles Sydney Gibbes', ['Charles Sydney Gibbes (19 January 1876 – 24 March 1963) was a British academic who from 1908 to 1917 served as the English tutor to the children of Emperor Nicholas II of Russia.', ' When Nicholas abdicated the throne in March 1917 Gibbes voluntarily accompanied the Imperial family into exile to the Siberian village of Tobolsk.', ' After the family was murdered in 1918 Gibbes returned to the United Kingdom and eventually became an Orthodox monk, adopting the name of \"Nicholas\" in commemoration of Nicholas II.', ' He died in 1963, and is buried at Headington cemetery, Oxford, Oxfordshire, England.']], ['October Manifesto', [\"The October Manifesto (Russian: Октябрьский манифест, Манифест 17 октября ), officially The Manifesto on the Improvement of the State Order (Манифест об усовершенствовании государственного порядка), is a document that served as a precursor to the Russian Empire's first constitution, which would be adopted the next year.\", ' The Manifesto was issued by Emperor Nicholas II, under the influence of Sergei Witte, on 30 October\\xa0[O.S. 17 October]\\xa01905 as a response to the Russian Revolution of 1905.', \" Nicholas strenuously resisted these ideas, but gave in after his first choice to head a military dictatorship, Grand Duke Nicholas, threatened to shoot himself in the head if the Tsar did not accept Witte's suggestion.\", ' Nicholas reluctantly agreed, and issued what became known as the October Manifesto, promising basic civil rights and an elected parliament called the Duma, without whose approval no laws were to be enacted in Russia in the future.', ' According to his memoirs Witte did not force the Tsar to sign the October Manifesto, which was proclaimed in all the churches.']], ['Russia–United Kingdom relations', [\"The Russia–United Kingdom relations (Russian: Российско-британские отношения ) is the relationship between the Russian Federation and the United Kingdom of Great Britain and Northern Ireland and it's overseas territories.\", ' Spanning nearly five centuries, it has often switched from a state of alliance to rivalry or even war.', ' The Russians and British were allies against Napoleon, and enemies in the Crimean War of the 1850s, and rivals in the Great Game for control of central Asia in the late 19th century.', ' They were allies again in World Wars I and II, although relations were strained by the Russian Revolution of 1917.', \" They were at sword's point during the Cold War (1947–91).\", ' Russian big businesses had strong connections with the City of London and British corporations during the late 1990s and 2000s.']], ['Ural State Mining University', ['Ural State Mining University (Russian: Уральский государственный горный университет ) is situated in Yekaterinburg, Russian Federation.', ' It was founded in 1914.', ' In 1917 Nicholas II signed an order titled \"On keeping of the Yekaterinburg Institute of Mines under the patronage of His Majesty the Emperor and on giving to this educational establishment the title of \"The Emperor Nicholas II Ural Institute of Mines\"\".']], ['Nicholas II of Russia', ['Nicholas II or Nikolai II (Russian: Николай II Алекса́ндрович , \"Nikolay II Aleksandrovich\" ; 18 May [O.S. 6 May] 1868 – 17 July 1918) was the last Emperor of Russia, ruling from 1 November 1894 until his forced abdication on 15 March 1917.', ' His reign saw the fall of the Russian Empire from being one of the foremost great powers of the world to economic and military collapse.', ' Due to the Khodynka Tragedy, anti-Semitic pogroms, Bloody Sunday, the violent suppression of the 1905 Revolution, the execution of political opponents and his perceived responsibility for the Russo-Japanese War, he was given the nickname Nicholas the Bloody by his political adversaries.', ' Soviet historiography portrayed Nicholas as a weak and incompetent leader, whose decisions led to military defeats and the deaths of millions of his subjects.']], ['Prince Andrew Romanov', ['Prince Andrew Andreyevich Romanov (born 21 January 1923) is a Russian American artist and author.', \" He is a grand-nephew of Russia's last Emperor, Nicholas II.\", ' Since December 31, 2016 he is a claimant to the headship of the Imperial House of Russia and President of the Romanov Family Association.', ' He is a great-great-grandson in the male-line of Emperor Nicholas I of Russia.']], ['Russian Revolution', ['The Russian Revolution was a pair of revolutions in Russia in 1917 which dismantled the Tsarist autocracy and led to the rise of the Soviet Union.', ' The Russian Empire collapsed with the abdication of Emperor Nicholas II and the old regime was replaced by a provisional government during the first revolution of February 1917 (March in the Gregorian calendar; the older Julian calendar was in use in Russia at the time).', \" Alongside it arose grassroots community assemblies (called 'soviets') which contended for authority.\", ' In the second revolution that October, the Provisional Government was toppled and all power was given to the soviets.']], ['Grand Duke Alexander Alexandrovich of Russia', ['Grand Duke Alexander Alexandrovich of Russia (Russian: Великий Князь Александр Александрович Романов; 7 June 1869 – 2 May 1870) was the infant son of Emperor Alexander III–the heir apparent, styled \"Tsesarevich\", to the Russian throne as the eldest living son of Emperor Alexander II–and his consort, Marie Fyodorovna of Russia.', \" He was Alexander and Marie's second child, second son, and the younger brother of the future Emperor Nicholas II.\", ' He died of meningitis in 1870, one month before his first birthday.', ' \"The doctors maintain he did not suffer, but we suffered terribly to see and hear him,\" his mother wrote to her own mother, Queen Louise of Denmark.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.524\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a89dd4d554299669944a5e3', 'answer': 'Blue Grass Airport', 'question': 'Flower Alley was bred by the trainer who was killed at what Fayette County, Kentucky airport?', 'supporting_facts': [['Flower Alley', 1], ['Blue Grass Airport', 0]], 'context': [['Fayette County School System (Georgia)', ['Fayette County School System is a public school district based in Fayetteville, Georgia, United States and covering residents of Fayette County.', ' The county is included in the Atlanta-Sandy Springs-Roswell Metropolitan Statistical Area.', ' The school system serves all of Fayette County.']], ['Flower Alley', ['Flower Alley (foaled May 7, 2002) is an American Thoroughbred racehorse.', ' He was bred at Bona Terra Farms by George Brunacini, who was killed in the August 27, 2006, crash of Comair Flight 5191 at Blue Grass Airport in Lexington, Kentucky.']], ['Manara, Ohio', ['Manara is an unincorporated community in Marion Township, Fayette County, Ohio, United States.', ' It is located at , at the intersection of Washington-Waterloo Road (Fayette County Highway 35) and Bloomingburg-New Holland Road (Fayette County Highway 27).']], ['Falmouth, Indiana', ['Falmouth is an unincorporated community in Fayette and Rush counties in the U.S. state of Indiana.', ' Located at the northeastern corner of Union Township and the southeastern corner of Washington Township in Rush County and along the northwestern edge of Fairview Township in Fayette County, it lies at the intersection of CR800E (Rush County) with CR600N (Rush County)/CR400N(Fayette County).', ' Falmouth sits northwest of Connersville and northeast of Rushville, the county seats of Fayette and Rush counties respectively.', ' Its elevation is 1,070\\xa0feet (326\\xa0m), and it is located at (39.7008798, -85.3010781).', ' Although Falmouth is unincorporated, it has a post office (located in Rush County), with the ZIP code of 46127.']], ['Oran, Iowa', ['Oran is an unincorporated community in southwestern Fayette County, Iowa, United States.', ' It lies along local roads southwest of the city of West Union, the county seat of Fayette County, and west of the city of Oelwein, the largest city in Fayette County.', ' Its elevation is 1,043\\xa0feet (318\\xa0m).', ' Although Oran is unincorporated, it has a post office with the ZIP code of 50664.']], ['Blue Grass Airport', ['Blue Grass Airport (IATA: LEX, ICAO: KLEX, FAA LID: LEX) is a public airport in Fayette County, Kentucky, 4 miles west of downtown Lexington.', ' Located among world-renowned horse farms and situated directly across from Keeneland Race Course, Blue Grass Airport is the primary airport serving central and eastern Kentucky.', ' More than 1.2 million passengers depart or arrive annually at Blue Grass Airport.', ' In 2016, the airport served 1,245,251 passengers via four major airline carriers: Allegiant, American Airlines, Delta Air Lines and United Airlines.']], ['Buena Vista, Fayette County, Ohio', ['Buena Vista is an unincorporated community in Green Township, Fayette County, Ohio, United States.', ' It is located at , at the intersection of Greenfield-Sabina Road (Fayette County Highway 5) and Stafford Road (Fayette County Highway 3), about 5 miles south of Washington Court House.', ' Rattlesnake Creek flows near the town site.']], ['Hurricane Hall', ['Hurricane Hall was built in the 1790s in Fayette County, Kentucky by David Laughed on the Lexington-Georgetown Pike.', ' Architecture historian Clay Lancaster describes it as \"the most engaging residence in Fayette County\".', ' The home is included in the National Register of Historic Places listings in Fayette County, Kentucky.']], ['Joseph A. Hardy Connellsville Airport', ['Joseph A. Hardy Connellsville Airport (ICAO: KVVS) is a public-use airport located four nautical miles (7 km) southwest of Connellsville in Dunbar Township, Fayette County, Pennsylvania, United States.', ' It is owned by the Fayette County Airport Authority and serves the south-eastern segment of the Pittsburgh metropolitan area.', ' The airport serves the general aviation community with no scheduled commercial airline service.']], ['Fayette County, Georgia', ['Fayette County is a county located in the north central portion of the U.S. state of Georgia.', ' As of the 2010 census, the population was 106,567.', ' Fayette County was established in 1821.', ' The county seat, Fayetteville, was established in 1823.', ' Much of Fayette County is bordered on the east side by the Flint River.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.525\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae199305542997b2ef7d20e', 'answer': 'Lacoste, France', 'question': 'In what european city is a location of the college from which the woman known as Comic Book Girl 19 received her degree?', 'supporting_facts': [['Comic Book Girl 19', 0], ['Comic Book Girl 19', 1], ['Savannah College of Art and Design', 0]], 'context': [['Comic book convention', ['A comic book convention or comic con is an event with a primary focus on comic books and comic book culture, in which comic book fans gather to meet creators, experts, and each other.', ' Commonly, comic conventions are multi-day events hosted at convention centers, hotels, or college campuses.', ' They feature a wide variety of activities and panels, with a larger number of attendees participating in cosplay than most other types of fan conventions. Comic book conventions are also used as a vehicle for industry, in which publishers, distributors, and retailers represent their comic-related releases.', ' Comic book conventions may be considered derivatives of science-fiction conventions, which began in the late 1930s.']], ['Thomas Sieverts', ['Thomas Sieverts (born 1934) is a German architect and urban planner.', ' He is the author of \"Zwischenstadt\" (1997; first published in English in 2000 as \"Cities without Cities: An interpretation of the Zwischenstadt\"), a book which addresses the decentralization of the compact historical European city and examines the new form of urbanity which has spread across the world describable as the urbanised landscape or the landscaped city.', ' Sieverts calls this the \"Zwischenstadt\", or \"in-between city\", as it exists between old historical city centres and open countrysides, between place as a living space and the non-places of movement, between small local economic cycles and the dependency on the world market.', ' In 2008 a group calling itself \"suddenly\" commissioned the American writer Diana George to make a new translation of \"Zwischenstadt\" which they published as \"Where We Live Now\" (the English phrase George chose as the translation of Sieverts\\'s neologism \"Zwischenstadt\").', ' In October 2008, Sieverts came to Portland, Oregon, on the occasion of the book\\'s publication to take part in a week-long symposium about his work, also called \"suddenly\".']], ['Sheena, Queen of the Jungle', ['Sheena, Queen of the Jungle is a fictional American comic book jungle girl heroine, originally published primarily by Fiction House.', ' She was the first female comic book character with her own title, with her 1937 (in Great Britain; 1938 in the United States) premiere preceding \"Wonder Woman\" #1 (cover-dated Dec. 1941).', ' Sheena inspired a wealth of similar comic book jungle queens.', ' She was predated in literature by Rima, the Jungle Girl, introduced in the 1904 William Henry Hudson novel \"Green Mansions\".', ' Sheena was ranked 59th in \"Comics Buyer\\'s Guide\"s \"100 Sexiest Women in Comics\" list.']], ['The Architecture of the City', ['The Architecture of the City (Italian: \"L\\'architettura della città\" ) is a seminal book of urban design theory by the Italian architect Aldo Rossi published in Padova in 1966.', ' The book marks the shift from the urban doctrines of modernism to a rediscovery of the traditional European city.']], ['Bratslav', ['Bratslav (Ukrainian: Брацлав ; Polish: \"Bracław\" ; Yiddish: בראָסלעוו\\u200e , \"Broslev\", today also pronounced Breslev or \"Breslov\" as the name of a Hasidic group, which originated from this town) is an urban-type settlement in Ukraine, located in Nemyriv Raion of Vinnytsia Oblast, by the Southern Bug river.', ' It is a medieval European city and a regional center of the Eastern Podolia region (see Bratslav Voivodeship) founded by government of the Crown of the Kingdom of Poland, which dramatically lost its importance during the 19th-20th centuries.', ' Population: \\u2009(2015 est.)']], ['Microcosm: Portrait of a Central European City', ['Microcosm: Portrait of a Central European City is a 2002 book by historians Norman Davies and Roger Moorhouse about the history of Wrocław, the largest city in western Poland.']], ['Metropolis Collectibles', ['Metropolis Collectibles is a famous rare comic book dealer of vintage American comics, primarily known for its large collection of comic books originally published in the 1930s, 1940s, 1950s, 1960s and 1970s.', \" Metropolis was founded in 1984 by Stephen Fishler, and merged companies in 1999 with Vincent Zurzolo, Jr., of Vincent's Collectibles.; Zurzolo said that as he found he could not compete with Fishler's business, merging the two made sense.\", ' The company is located on Broadway in New York City, and the comic book showroom allows viewings by appointment only.', ' Over the years, Metropolis Collectibles has grown from being a comic-book mail-order company to maintaining a major online retail presence.', ' In addition to being comic book buyers and comic book sellers, Metropolis also gives comic book appraisals and provides comic book valuation services of rare, old out-of-print comics.', ' Metropolis Collectibles has obtained a variety of notable classic comic book collections over the years, or \"pedigrees\", including the Crowley Collection, the Allentown Pedigree, the D-Copy Collection, and the Northford Collection.', ' In August 2014, the company was able to purchase a near-mint copy of \"Action Comics #1\" (CGC 9.0) for $3.2 million in an auction on eBay.']], ['Savannah College of Art and Design', ['Savannah College of Art and Design (SCAD), is a private, nonprofit, accredited university with locations in Savannah, Georgia; Atlanta, Georgia; Hong Kong; and Lacoste, France.']], ['Parks and open spaces in Birmingham', ['Birmingham has 571 parks totalling over 3500 ha of public open space, more than any other equivalent sized European city.', \" The centrepieces of Birmingham's park system are the five Premier Parks.\", ' Ten parks have received the prestigious Green Flag Award.', ' The city also has five local nature reserves, one national nature reserve and a number of Wildlife Trust nature reserves.']], ['Comic Book Girl 19', ['Danika Lee Massey, also known as Comic Book Girl 19 or CBG19, is a YouTube personality known for her commentaries on comics, films, books, and television shows.', ' She has a degree in sequential art from the Savannah College of Art and Design.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.526\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abba3cd554299642a094aee', 'answer': 'six', 'question': 'How many different schools does the university, in which Andrew J. Elliot is a professor of psychology, have?', 'supporting_facts': [['Andrew J. Elliot', 0], ['University of Rochester', 2]], 'context': [['Archives of Scientific Psychology', ['Archives of Scientific Psychology is an open access academic journal published by the American Psychological Association.', ' The journal publishes a wide variety of articles pertaining to the many different sub-fields of psychology, such as neuroscience and political psychology.', ' The journal includes articles that cover the many different research methodologies employed by psychologists.', ' The current editors-in-chief are Cecil R. Reynolds (Texas A&M University) and Gary R. VandenBos (American Psychological Association).']], ['Neigong', ['Neigong, also spelled \"nei kung\", \"neigung\", or \"nae gong\", refers to any of a set of Chinese breathing, meditation and spiritual practice disciplines associated with Daoism and especially the Chinese martial arts.', ' Neigong practice is normally associated with the so-called \"soft style\", \"internal\" or neijia 內家 Chinese martial arts, as opposed to the category known as waigong 外功 or \"external skill\" which is historically associated with shaolinquan or the so-called \"hard style\", \"external\" or wàijiā 外家 Chinese martial arts.', ' Both have many different schools, disciplines and practices and historically there has been mutual influence between the two and distinguishing precisely between them differs from school to school.']], ['Andrew J. Offutt', ['Andrew Jefferson Offutt (August 16, 1934 – April 30, 2013) was an American science fiction and fantasy author.', ' He wrote as Andrew J. Offutt, A. J. Offutt, and Andy Offutt.', ' His normal byline, andrew j. offutt, has all his name in lower-case letters.', ' He also wrote erotica under seventeen different pseudonyms, principally John Cleve, John Denis, Jeff Morehead, and Turk Winter.', ' He is the father of novelist Chris Offutt and professor Jeff Offutt.']], ['Affix grammar over a finite lattice', ['In linguistics, the affix grammars over a finite lattice (AGFL) formalism is a notation for context-free grammars with finite set-valued features, acceptable to linguists of many different schools.']], ['Hojōjutsu', ['Hojōjutsu (捕縄術), or Torinawajutsu (捕縄術), or just Nawajutsu (縄術), is the traditional Japanese martial art of restraining a person using cord or rope (said \"nawa\" 縄 in Japanese).', ' Encompassing many different materials, techniques and methods from many different schools, Hojōjutsu is a quintessentially Japanese art that is a unique product of Japanese history and culture.']], ['Michael Bayne', ['Michael Bayne is an athletic coach who has led teams in many sports, and in schools all across North and South Carolina.', ' He served as the Head Golf Coach and Special Teams Coordinator at Brevard College from 2006 until 2010, where he then worked as the Head Track, Cross Country and Lacrosse Coach and Special Teams Coordinator for North Greenville University.', ' He grew up in South Carolina, receiving his B.A. Degree from University of South Carolina in 1980.', ' Bayne has provided services as the Head and Assistant Coach for many different schools and team sports, from 1984 to the present.', ' In addition to his coaching career, Michael Bayne has been an upstanding educator and administrator.']], ['Jennifer McFalls', ['Jennifer Yvonne McFalls (born November 10, 1971) is a retired professional softball player who played for Texas A&M and then went on to the U.S. National Softball Team.', ' After her years playing softball McFalls decided to become a coach with her first position as the assistant coach at Texas A&M.', ' Mcfalls continued to coach for many years with several different schools at many different competitive levels.', ' She was the head coach of the National Pro Fastpitch professional softball team, the Dallas Charge for their inaugural season.']], ['University of Rochester', ['The University of Rochester ( U of R or UR) frequently referred to simply as Rochester, is a private, nonsectarian, research university in Rochester, New York.', ' The university grants undergraduate and graduate degrees, including doctoral and professional degrees.', ' The university has six schools and various interdisciplinary programs.']], ['Andrew J. Elliot', ['Andrew J. Elliot (born 1962) is a professor of psychology at the University of Rochester.', ' His research on the hierarchical model of approach and avoidance motivation focuses on combining classic and contemporary methods to test various theories.', \" Elliot's work in social psychology is cited frequently by those in the field, causing him to be named one of Thomson Reuters' ISI Highly Cited for the Social Sciences in 2010.\"]], ['List of yoga schools', ['Yoga, rather than being the name for a singular lineage or even a specific practice, is a bracket term that covers a number of methodologies, each with a number of schools.', ' Within the major branches of yoga such as haṭha, lāya, rāja, jñāna, and bhakti there are many different schools and lineages, both extant and defunct.', ' Since the late 19th century, a great number of distinct new styles of \"Yoga\" have been introduced by individual teachers.', ' There are also a number of schools and traditions that are occasionally referred to as yoga or yogic for their similar practices despite having no foundation in the Indian tradition such as Shin Shin Tōitsu-dō, and Daoyin.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.526\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac540365542994611c8b470', 'answer': 'Loughborough University', 'question': \"Loughborough Students' Union serves the students at what public research university that has been a university since 1966, but the institution dates back to 1909?\", 'supporting_facts': [[\"Loughborough Students' Union\", 0], ['Loughborough University', 0], ['Loughborough University', 1]], 'context': [['Loughborough Students (Lightning) RUFC', [\"Loughborough Students (Lightning) Rugby Union Football Club are a women's rugby union club based in Loughborough, Leicestershire, England.\", \" They are the women's team of Loughborough Students RUFC and Loughborough University.\", ' In 2017, they were selected as a franchise for the inaugural Premier 15s season.']], ['Haslegrave Ground', ['Haslegrave Ground is a cricket ground in Loughborough, Leicestershire.', ' The ground is based at Loughborough University.', ' The first recorded match on the ground was in 1988, when Loughborough Students played the Marylebone Cricket Club.', \" The ground held 2 Women's One Day Internationals in 2008, when England women played West Indies women in both matches.\"]], ['Henry Weir', ['Henry Weir (born 13 February 1990) is a British field hockey player.', ' Graduated from Loughborough University with a degree in sports science.', ' He made his international debut against India in December 2012 at the Melbourne Champions Trophy only 11 years after he first started playing hockey for the Crewe Vagrants.', \" Weir competed for England in the men's hockey tournament at the 2014 Commonwealth Games where he won a bronze medal.\", ' Has played for Loughborough Students, Brooklands MU, Crewe Vagrants, Reigate Priory and currently Wimbledon.']], [\"Loughborough Students' Union\", [\"Loughborough Students' Union (otherwise known as LSU) is the students' union serving members from Loughborough University, Loughborough College and the RNIB College Loughborough.\"]], ['Ohio Union', ['The Ohio Union serves as a student activity center for students of The Ohio State University.', ' When the Union was established in 1910, it was the first student union at a public university.', ' The Ohio Union provides facilities for student activities, organizations/events, and campus and community interaction.', ' Many student services and programs are housed in the union, along with dining and recreational facilities.', ' It also serves as the home base for the D-Tix program, which provides discounted tickets to students.', ' On March 29, 2010, the current Ohio Union was erected.']], ['Loughborough University', ['Loughborough University (abbreviated as Lough for post-nominals) is a public research university located in the market town of Loughborough, Leicestershire, in the East Midlands of England.', ' It has been a university since 1966, but the institution dates back to 1909, when the then Loughborough Technical Institute began with a focus on skills and knowledge which would be directly applicable in the wider world.', ' In March 2013, the university announced it had acquired the former broadcast centre at the Queen Elizabeth Olympic Park which opened as a second campus in 2015.', ' It was a member of the 1994 Group until the group was dissolved in November 2013.']], ['University of Nairobi', ['The University of Nairobi (UoN) is a collegiate research university based in Nairobi.', ' It is one of the largest universities in Kenya.', ' Although its history as an educational institution dates back to 1956, it did not become an independent university until 1970.', ' In this year, the University of East Africa was split into three independent universities: Makerere University in Uganda, the University of Dar es Salaam in Tanzania, and the University of Nairobi.']], [\"Loughborough Students' Hockey Club\", [\"Loughborough Students' Hockey Club is a field hockey club based in Loughborough, England.\", ' The home ground is at Loughborough University Campus.', ' The club is the country’s leading student hockey club and has produced a number of international players.']], ['Loughborough University F.C.', ['Loughborough University Football Club (also known as Loughborough Students Football Club) is an English football club representing Loughborough University, based in Loughborough, Leicestershire.', ' The club are currently members of the Midland League Premier Division and play at the Loughborough University Stadium.']], ['Loughborough Students RUFC', ['Loughborough Students Rugby Union Football Club is the rugby club that represents Loughborough University in rugby union competition.', ' Of the British universities, Loughborough has unparalleled success, having won the BUCS championship (in its former guises as the BUSA and UAU championship) on twenty-seven occasions.', ' It fields sides in the BUCS league, (inter-university) and in the third tier of the English rugby union system, National League 1.', ' The club has fielded over seventy internationals (male and female), many of whom won caps while playing for the club.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.527\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae789615542997ec2727695', 'answer': 'Martha Coolidge', 'question': 'Kam Heskin plays Paige Morgan in a 2004 film directed by who?', 'supporting_facts': [['Kam Heskin', 2], ['The Prince and Me', 0]], 'context': [['True True Lie', ['True True Lie is 2006 thriller film directed by Eric Styles.', ' The film follows Dana, who, after 12 years in an asylum, is reunited with her family and childhood friends Nathalie and Paige.', ' Dana slowly begins to realize that the events that led to her stay there may not have been imaginary.', ' \"True True Lie\" stars Jaime King as Nathalie, Lydia Leonard as Dana and Annabelle Wallis as Paige.']], ['The Prince &amp; Me 2: The Royal Wedding', ['The Prince and Me 2: The Royal Wedding is a 2006 romantic comedy film and the sequel to the 2004 film \"The Prince and Me\" and was released direct-to-video.', ' Directed by Catherine Cyran, the film features Luke Mably reprising his role as King Edvard of Denmark, with Kam Heskin replacing Julia Stiles as Paige Morgan and Clemency Burton-Hill as newcomer Princess Kirsten of Norway.']], ['Turning Paige', ['Turning Paige is a 2001 Canadian drama film directed by Robert Cuffley.', \" The film focuses on the life of Paige (Katharine Isabelle) and her family a she comes to terms with tragedy in the family's past.\", ' The film\\'s title refers to the phrase \"turn the page\" as the central character must come to terms with her past and put it behind her if she is to move on in life.']], ['Sunset at Chaophraya (2013 film)', ['Sunset at Chaophraya (Thai: คู่กรรม , \"Khu Kam\") in 2013 romantic-war-drama film directed by Kittikorn Liasirikun.', ' Adapted from the novel \"Khu Kam\" by Thommayanti, the story is a love triangle, set in World War II-era Thailand, and depicts the star-crossed romance between an Imperial Japanese Navy officer and a Thai woman who is involved with the Free Thai resistance.', ' It was released on April 4, 2013, One of top five box office hits in (Thailand film) in 2013']], ['Super Size Me 2: Holy Chicken!', ['Super Size Me 2: Holy Chicken!', ' is a 2017 American documentary film directed by Morgan Spurlock.', ' A sequel to the 2004 film \"Super Size Me\", it explores the ways in which the fast food industry has rebranded itself as healthier since his original film through the process of Spurlock working to open his own fast food restaurant, thus exposing some of the ways in which that rebranding is more perception than reality.']], ['Sunset at Chaophraya (1996 film)', ['Sunset at Chaophraya (Thai: คู่กรรม , \"Khu Kam\") is a 1996 romantic-drama film directed by Euthana Mukdasanit.', ' Adapted from the novel \"Khu Kam\" by Thommayanti, the story is a love triangle, set in World War II-era Thailand, and depicts the star-crossed romance between an Imperial Japanese Navy officer and a Thai woman who is involved with the Free Thai resistance.']], ['The Prince and Me', ['The Prince and Me is a 2004 romantic comedy film directed by Martha Coolidge, and starring Julia Stiles, Luke Mably, and Ben Miller, with Miranda Richardson, James Fox, and Alberta Watson.', ' The film focuses on Paige Morgan, a pre-med college student in Wisconsin, who is pursued by a prince posing as a normal college student.']], ['Kam Heskin', ['Kam Heskin (born Kam Erika Heskin on May 8, 1973) is an American actress.', ' She began her career playing Caitlin Richards Deschanel on the NBC daytime soap opera \"Sunset Beach\" (1998–1999), before appearing in films \"Planet of the Apes\" (2001 and \"Catch Me If You Can\" (2002).', ' Heskin went to play Elizabeth Bennet in the 2003 independent film \"\", and Paige Morgan in the \"The Prince and Me\" film franchise (2006–2010).']], ['R-Point', ['R-Point () is a 2004 Korean horror film written and directed by Kong Su-chang.', ' Set in 1972 Vietnam, during the Vietnam War, it stars Kam Woo-sung and Son Byong-ho as members of the South Korean military in Vietnam.', ' Most of the movie was shot in Cambodia.', ' Bokor Hill Station plays a prominent part of the movie, in this case doubling as a colonial French plantation.']], ['Pride &amp; Prejudice: A Latter-Day Comedy', [\"Pride & Prejudice: A Latter-Day Comedy is a 2003 independent film adaptation of Jane Austen's novel set in modern-day Provo, Utah.\", ' The film received mixed reviews, with more negative reviews than positive.', ' Critics accused the film of its poor editing and its rough application of the story to modern life.', ' Positive reviews praised Kam Heskin\\'s performance as Elizabeth and enjoyed that the film was \"cute\".', \" Although the film included aspects of LDS culture, most critics agreed that the film's connection with LDS culture was trivial, making the film more universally accessible to viewers.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.528\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a75e67d5542992db9473709', 'answer': 'New York and New Jersey campaign', 'question': 'What was a series of battles during the Revolutionary War, for control of New York City and the state of New Jersey, fought on October 28, 1776 near White Plains, New York?', 'supporting_facts': [['Battle of White Plains', 0], ['New York and New Jersey campaign', 0]], 'context': [['New York and New Jersey campaign', ['The New York and New Jersey campaign was a series of battles for control of New York City and the state of New Jersey in the American Revolutionary War between British forces under General Sir William Howe and the Continental Army under General George Washington in 1776 and the winter months of 1777.', ' Howe was successful in driving Washington out of New York City, but overextended his reach into New Jersey, and ended the active campaign season in January 1777 with only a few outposts near the city.', ' The British held New York harbor for the rest of the war, using it as a base for expeditions against other targets.']], ['Ambush of Geary', ['The Ambush of Geary was a skirmish of the American Revolutionary War fought on 14 December 1776 near Ringoes in Amwell Township, Hunterdon County, New Jersey.', ' Cornet Francis Geary, the leader of a company of dragoons, was shot in an ambush set up by local militiamen.']], ['USS White Plains (AFS-4)', ['USS \"White Plains\" (AFS-4) was the fourth \"Mars\"-class combat stores ship of the United States Navy.', ' The ship was named after the city of White Plains, New York, scene of the Battle of White Plains during the American Revolutionary War.']], ['Battle of Mamaroneck', ['The Battle of Mamaroneck was a skirmish in the New York and New Jersey campaign of the American Revolutionary War fought on October 22, 1776, at Mamaroneck, Westchester County, New York.', \" Following the retreat of George Washington's army to White Plains, British General William Howe landed troops in Westchester County, intending to cut off Washington's escape route.\", ' To cover the eastern flank of his army, Howe ordered Major Robert Rogers and his Rangers to seize the village of Mamaroneck which had been recently abandoned by the Continental army.', ' On the night of October 22, 750 men under Colonel John Haslet attacked the British encampment.', \" Haslet's men achieved complete surprise, but Rogers' Rangers rallied and drove off the attackers.\"]], ['Pelham Parkway (neighborhood), Bronx', ['Pelham Parkway is a working- and middle-class residential neighborhood geographically located in the center of the Bronx, a borough of New York City in the United States.', ' The neighborhood is part of Bronx Community Board 11.', ' Its boundaries, starting from the north and moving clockwise are: Waring Avenue to the north, the IRT Dyre Avenue Line tracks ( trains ) to the east, Neill Avenue to the South, and Bronx River Parkway to the west.', ' White Plains Road is the primary commercial thoroughfare through Bronx Park East.', ' The local subway line is the IRT White Plains Road Line ( trains ) operating along White Plains Road.', ' Zip codes include 10461 and 10462.', ' The area is patrolled by the New York City Police Department 49th Precinct located at 2121 Eastchester Road in the Morris Park section of the Bronx.']], ['Battle of White Plains', ['The Battle of White Plains was a battle in the New York and New Jersey campaign of the American Revolutionary War fought on October 28, 1776, near White Plains, New York.', \" Following the retreat of George Washington's Continental Army northward from New York City, British General William Howe landed troops in Westchester County, intending to cut off Washington's escape route.\", ' Alerted to this move, Washington retreated farther, establishing a position in the village of White Plains but failed to establish firm control over local high ground.', \" Howe's troops drove Washington's troops from a hill near the village; following this loss, Washington ordered the Americans to retreat farther north.\"]], ['New York State Route 125', ['New York State Route\\xa0125 (NY\\xa0125) is a 7.50 mi north–south state highway located within Westchester County, New York, in the United States.', ' The route begins at an intersection with U.S. Route\\xa01 (US\\xa01) in the town of Mamaroneck and ends at a junction with NY\\xa022 in the city of White Plains.', ' A section of the route in the city of White Plains is maintained by Westchester County and co-designated as County Route\\xa026 (CR\\xa026).', ' A second county-owned segment exists along the New Rochelle–Scarsdale line as County Route\\xa0129.', ' Both numbers are unsigned.', ' NY\\xa0125 was assigned as part of the 1930 renumbering of state highways in New York, initially extending from US\\xa01 to Mamaroneck Avenue in White Plains.', ' It was extended north to NY\\xa022 in the mid-1930s.']], ['City Center at White Plains', ['City Center at White Plains is a large mixed-use development shopping complex in downtown White Plains, New York.', ' It features two 35-story apartment and condominium towers, 600000 sqft of retail, restaurant and entertainment space and new parking facilities.', \" City Center's opening in 2003 marked the beginning of a new downtown development renaissance, and with the improving economy and healthy office leasing activity, White Plains entered the new millennium as the leading retail and office center in Westchester County.\", ' The City Center has also become a known spot where members of the New York Knicks hang out during their time off since their practice facility is in Greenburgh, New York, which is just 14 minutes away.']], ['James Linn', ['James Linn (1749 – January 5, 1821) was a United States Representative from New Jersey.', ' Born in Bedminster Township, he pursued preparatory studies and graduated from Princeton College in 1769.', ' He studied law, was admitted to the bar in 1772 and commenced practice in Trenton.', ' He returned to Somerset County and was judge of the Court of Common Pleas; he was a member of the Provincial Congress of New Jersey in 1776.', ' During the Revolutionary War he served as captain in the Somerset County Militia in 1776, and first major from 1776 to 1781.', ' He was a member of the New Jersey Legislative Council (now the New Jersey Senate) in 1777, and returned to Trenton; he served in the New Jersey General Assembly in 1790 and 1791, and from 1793 to 1797 was again a member of the State Council serving as Vice-President of Council in 1796-97.']], ['10th Continental Regiment', ['The 10th Continental Regiment was a unit of the Connecticut Line in the 1776 establishment of the Continental Army.', \" It began as Parson's Connecticut Regiment (also known as the 6th Connecticut Provincial Regiment), which was part of the 1775 establishment, and was commanded by Colonel Samuel Holden Parsons until his promotion to brigadier general.\", ' It was first active during the Siege of Boston, and then in preparing the defenses of New York City.', ' After Parsons was promoted in August 1776, command came to John Tyler, who was promoted to colonel at that time.', ' The regiment fought in the Battle of Long Island, and was part of the panicked retreat after the British landing on Manhattan.', ' Although the regiment was present with the army at White Plains, New York in October 1776, it did not participated in the battle fought there.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.529\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8a5a1955429930ff3c0da7', 'answer': '1983', 'question': 'In what year was the Sayrevill, New Jersey rock band that Frankie LaRocka played drums for formed?', 'supporting_facts': [['Frankie LaRocka', 1], ['Bon Jovi', 1]], 'context': [['The Rats (American band)', ['The Rats were an American garage punk band from Portland, Oregon, formed by Fred Cole previously of the garage rock band, The Lollipop Shoppe.', ' Cole played guitar and sang, his wife, \"Toody\" played bass and sang, and initially Rod Rat played drums.', ' Their sound was a raw mix of punk rock with occasional country touches.', \" Their self-titled debut album appeared on Cole's Whizeagle label in 1980.\", ' Soon after, Rod Rat left the band, though he guested on the 1981 follow-up \"Intermittent Signals\" before his death by suicide.', ' (Prior to his suicide Rod Rat (aka Rod Hibbert) also played drums in 1980-81 for Portland power pop band Domino Theory).', ' Sam Henry, formerly of the Wipers, played drums on this LP but left to join another Portland band, Napalm Beach.', ' Louis Samora was on the drum throne for the 1983 album \"In a Desperate Red\", still on Whizeagle.', ' Samora left in 1984 to concentrate on his rockabilly band, The Jackals.', ' The band broke up, but Bill Barker of Profile Studios in Vancouver, British Columbia convinced the band to reunite for a single.', ' It appeared under the band name The Desperate Edge later in 1984.', ' Soon after, Cole assembled a country band, Western Front, and he and Toody later reunited in Dead Moon.', \" The Rats' records have long been out of print and sell for high prices on eBay.\", \" In 2008, Portland's Mississippi Records reissued the first album on vinyl.\"]], ['Bon Jovi', ['Bon Jovi is an American rock band from Sayreville, New Jersey.', ' Formed in 1983, Bon Jovi consists of lead singer and namesake Jon Bon Jovi, pianist and keyboardist David Bryan, drummer Tico Torres, lead guitarist Phil X, and bassist Hugh McDonald.', \" The band's lineup has remained mostly static during its history, with the only exceptions being the 1994 dismissal of bass player Alec John Such, who was unofficially replaced by Hugh McDonald, and the departure of longtime guitarist and co-songwriter Richie Sambora in 2013.\", ' Phil X and McDonald both became official members in 2016.']], ['A Date with The Smithereens', ['A Date with the Smithereens is the fifth album by the New Jersey rock band The Smithereens, released in 1994.', ' It is seen as a stylistic departure by the band, as \"A Date\" is mainly influenced by hard rock and heavy metal, while previous albums have leaned more towards straightforward pop rock.', ' The title is meant to be ironic as nearly all songs on the album are hate-inspired.']], ['The Rosebuds', ['The Rosebuds are an indie rock band from Raleigh, North Carolina, United States.', \" (Not to be confused with the vocal group of the same name who recorded for George Goldner's Gee Records in the 1950s.)\", ' Its current members are Ivan Howard (vocals/guitar/drums/bass/keyboards/programming) and Kelly Crisp (vocals/keyboard/drums/guitar/accordion).', ' Billy Alphin had played drums on the album \"The Rosebuds Make Out\", Wes Phillips played drums on the E.P.\"Unwind\", Lee Waters played drums on the record \"Birds Make Good Neighbors\", and Matt McCaughan played the drums on \"Night of the Furies\", \"Life Like\" and \"Loud Planes Fly Low.\"', ' Rob Lackey was behind the drum kit during the taping of \"The Rosebuds Live at the Cats Cradle\", a charity only recording sold at \"cytunes.org\" to help fight for the prevention of cancer, as well as on the track \"Second Birds of Paradise\" on \"Loud Planes Fly Low.\"']], ['The Baby Namboos', ['The Baby Namboos were a 1990s British trip hop band.', ' One of their members, Mark Porter, is the cousin of Tricky, who produced some tracks on their debut album.', ' Their song \"Late Night Antics\" inspired a New Jersey rock band to call themselves Mister Behavior, a persona mentioned in the song.']], ['Royal Teens', ['The Royal Teens was a New Jersey rock and roll band that formed in 1956, which was composed of Bob Gaudio on piano, Tom Austin on drums, Billy Dalton on guitar, and Billy Crandall on saxophone.', ' The group is best known for its single \"Short Shorts,\" which was a #3 hit in the United States in 1958.', ' The follow-up single, 1959\\'s \"Believe Me,\" hit #26.', ' They never recorded an album, and broke up in 1965.']], ['Frankie LaRocka', ['Frankie LaRocka (April 17, 1954 – May 12, 2005), born Frank LaRocca, was an American rock musician and producer.', ' He was a member of Scandal, playing drums on their debut EP, and, at various stages, played the drums with Bon Jovi, David Johansen, Bryan Adams, and John Waite.', ' In the 1990s, he became an A&R man and producer with his most notable work being with the Spin Doctors on their breakthrough album \"Pocket Full of Kryptonite\".']], ['The Del-Aires', ['The Del-Aires were a Paterson, New Jersey rock band of the 1960s.', ' They were featured as themselves in Del Tenney\\'s 1964 B-movie beach party film, \"The Horror of Party Beach.\"', ' For the film, Gary Robert Jones and Ronnie Linares wrote one song together, \"Drag,\" and one song each: \"Wigglin\\' Wobblin\\'\" (Jones) and \"Elaine\" (Linares).', ' The Del-Aires performed all six songs in the film, which included \"Joy Ride\", \"The Zombie Stomp\" and \"You Are Not a Summer Love.\"', ' Following his stint with the Del-Aires, saxophonist/guitarist/keyboardist Bobby Osborne was a member of the band Gas Mask, perhaps best known for having their first (and only) album, \"Their First Album,\" produced by Teo Macero.']], ['Sean Aaberg', ['Sean Aaberg was born in Oakland, California.', ' He is the son of Philip Aaberg and LouAnn Lucke.', \" He grew up in Oakland, attending Bishop O'Dowd High School and briefly attended the California College of the Arts.\", ' As a kid, he liked \"anything weird, nasty and old\", including cheap magazines and bought thousands of them, he said in an interview.', ' He and his friends read \"Mad\", drew comics, liked to listen to Cheech and Chong and the Ramones on the Dr. Demento show, and later discovered the Church of the Subgenius.', ' He admired the movies of Ralph Bakshi.', ' The interviewer noted that Aaberg writes in ALL CAPS.', ' Sean founded and played drums in the Oakland-based Hardcore Punk band The Masked Men, played drums for Baltimore-based Anarcho Punk band A//Political and founded and played drums for Eugene-based Rock and Roll band The Latrines.']], ['Live at the Court: Greatest Hits and More', ['Live at the Court: Greatest Hits and More is the ninth album from New Jersey rock band The Smithereens.', ' It was recorded live at the Court Tavern in New Brunswick, New Jersey.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.530\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7bc0755542995eb53be9a5', 'answer': 'The Dark Tower', 'question': 'The series of novels that reference numerous locations and incorporates themes from multiple genres is titled what?', 'supporting_facts': [['Places in The Dark Tower series', 0], ['The Dark Tower (series)', 0]], 'context': [['Idyllen', ['Idyllen (\"Idylls\"), Op.', ' 95, is a waltz composed by Johann Strauss II in 1851.', ' It was composed for a \"Grand Summer Festival Soiree\" in the Vienna Volksgarten, with the composer conducting the Strauss Orchestra in its first performance.', ' The fifth waltz melody of the work incorporates themes from the popular song \"O Madchen mein unter\\'m Hollerstock\" (\"O maiden mine beneath the rose-tree\").', ' The \"Theater Zeitung\" commented on the Strauss waltz that \"it is most original and displays a multitude of piquant dance rhythms which are instrumented with much spirit and, amidst tumultuous applause from the select and numerous public, had to be played \"da capo\".\"']], ['Bas-Lag', [\"Bas-Lag is the fictional world in which several of English author China Miéville's novels are set.\", ' Bas-Lag is a world where both magic (referred to as \"thaumaturgy\") and steampunk technology exist, and is home to many intelligent races.', ' It is influenced by the themes and tropes of multiple genres of science fiction, fantasy, and horror.']], ['SLA Industries', ['SLA Industries (pronounced \"slay\") is a role-playing game first published in 1993 by Nightfall Games in Glasgow, Scotland.', ' The game is set in a dystopian far-flung future in which the majority of the known universe is either owned or indirectly controlled by the eponymous corporation \"SLA Industries\" and incorporates themes from the cyberpunk, horror, and conspiracy genres.']], ['Places in The Dark Tower series', ['\"The Dark Tower\" series of novels, by Stephen King, contain references to numerous locations.', ' Some of those locations are listed below.']], [\"I'm Out\", ['\"I\\'m Out\" is a song recorded by American recording artist Ciara, for her eponymously titled fifth studio album (2013), featuring guest vocals from Trinidadian rapper Nicki Minaj.', ' It was written by Ciara, Minaj and Rock City brothers, Timothy and Theron Thomas.', ' Rocky City and The Co-Captains provided production.', ' \"I\\'m Out\" made its premiere on May 22, 2013 on Ciara\\'s official SoundCloud account and was serviced to urban radio in the United States on June 3, 2013 as the second single from \"Ciara\" through Epic Records.', ' Lyrically, it incorporates themes of breakups and making an ex-boyfriend regret leaving.']], ['Lila Downs', ['Ana Lila Downs Sánchez, best known as Lila Downs (born September 9, 1968) is a Mexican-American singer-songwriter and actress.', ' She performs her own compositions and the works of others in multiple genres, as well as tapping into Mexican traditional and popular music.', ' She also incorporates indigenous Mexican influences and has recorded songs in many indigenous languages such as Mixtec, Zapotec, Mayan, Nahuatl and Purépecha.', ' Born and raised in Oaxaca, she primarily studied at the Institute of Arts by Oaxaca and briefly attended University of Minnesota, before withdrawing to focus on her musical career.', ' She soon began performing in the traditional music scene of Oaxaca City.']], ['The Dark Tower (series)', ['The Dark Tower is a series of eight books written by American author Stephen King that incorporates themes from multiple genres, including dark fantasy, science fantasy, horror, and Western.', ' It describes a \"gunslinger\" and his quest toward a tower, the nature of which is both physical and metaphorical.', \" The series, and its use of the Dark Tower, expands upon Stephen King's multiverse and in doing so, links together many of his other novels.\", ' King has described the series as his \"magnum opus\".', \" In addition to the eight novels of the series proper that comprise 4,250 pages, many of King's other books relate to the story, introducing concepts and characters that come into play as the series progresses.\"]], ['Genre', ['Genre ( , or ; from French \"genre\" ] , \"kind\" or \"sort\", from Latin \"genus\" (stem \"gener-\"), Greek γένος, \"génos\") is any form or type of communication in any mode (written, spoken, digital, artistic, etc.) with socially-agreed upon conventions developed over time.', ' Genre is most popularly known as a category of literature, music, or other forms of art or entertainment, whether written or spoken, audio or visual, based on some set of stylistic criteria, yet genres can be aesthetic, rhetorical, communicative, or functional.', ' Genres form by conventions that change over time as new genres are invented and the use of old ones is discontinued.', ' Often, works fit into multiple genres by way of borrowing and recombining these conventions.', ' Stand alone texts, works, or pieces of communication may have individual styles, but genres are amalgams of these texts based on agreed upon or socially inferred conventions.', ' Some genres may be rigid with strictly adhered to guidelines while others may be very flexible.']], ['El León (album)', ['El León (Spanish for \"The Lion\") is the seventh studio album by the Argentine ska and reggae band Los Fabulosos Cadillacs.', ' Released in 1992, the album combines multiple genres and incorporates Caribbean rhythms as well as salsa, calypso and reggae influences.', ' It is a notable album in the history of Argentine rock.']], ['Leslie-Lohman Museum of Gay and Lesbian Art', ['The Leslie-Lohman Museum of Gay and Lesbian Art, operated by the Leslie-Lohman Gay Art Foundation, is a visual art museum in SoHo, Manhattan, New York City.', ' It mainly collects, preserves and exhibits visual arts created by LGBTQ artists or art about LGBTQ themes, issues, and people.', ' The Museum offers exhibitions year-round in numerous locations and owns more than 22,000 objects, including, paintings, drawings, photography, prints and sculpture.', ' It has been recognized as one of the oldest arts groups engaged in the collection and preservation of gay art.', ' In May 2011, the Foundation was awarded Museum status by the New York State Board of Regents.', ' The Museum is a member of the American Alliance of Museums and operates pursuant to their guidelines.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.531\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae7535c5542997b22f6a6d8', 'answer': '1998', 'question': 'The 53rd National Hockey League All-Star Game took place at the indoor arena that was completed in what year?', 'supporting_facts': [['53rd National Hockey League All-Star Game', 0], ['BB&amp;T Center (Sunrise, Florida)', 0], ['BB&amp;T Center (Sunrise, Florida)', 2]], 'context': [['NHL All-Star Skills Competition', ['The NHL All-Star Game SuperSkills Competition, originally known as the National Hockey League All-Star Skills Competition, is an event on the night preceding the All-Star Game.', ' Started at the 41st National Hockey League All-Star Game in Pittsburgh in 1990, the NHL uses the event to showcase the talents of its all-star participants.', ' Events include accuracy shooting, fastest skater, Skills Challenge Relay, hardest shot, Breakaway Challenge, and an Elimination Shootout.', ' The All-Star teams select representatives for each event, with points awarded to the winning team.']], ['BB&amp;T Center (Sunrise, Florida)', ['The BB&T Center (previously known as the National Car Rental Center, Office Depot Center, and BankAtlantic Center) is an indoor arena located in Sunrise, Florida.', ' It is home to the Florida Panthers of the National Hockey League.', ' It was completed in 1998, at a cost of US$185 million, almost entirely publicly financed, and features 70 suites and 2,623 club seats.']], ['63rd National Hockey League All-Star Game', ['The 63rd National Hockey League All-Star Game, also known as the 2018 NHL All-Star Game, will be held at Amalie Arena in Tampa, Florida, home of the Tampa Bay Lightning on January 28, 2018.', ' Tampa last held the NHL All Star Game in 1999.', ' The All-Star Game will be played in lieu of NHL participation in the 2018 Olympics, as the NHL Board of Governors ruled against interrupting the season to send players to PyeongChang from 10–25 February.']], ['NBL Canada All-Star Game', ['The NBL Canada All-Star Game is an exhibition game hosted by the National Basketball League of Canada (NBL), currently matching a group of players from the Central Division against a group from the Atlantic Division.', ' The first All-Star game took place at the Halifax Metro Centre in Halifax, Nova Scotia on April 1, 2012.', ' In the inaugural game, the format did not take divisions into account and randomly divided players under captains Joey Haywood and Eddie Smith.', \" As of 2014, the players are decided on the coaches' votes.\", ' No All-Star game took place for the 2014–15 NBL Canada season.', ' The game takes place alongside multiple other competitions, together known as All-Star Weekend.', ' These competitions include the Three-Point Long Distance Shootout and the Slam Dunk Championship.', ' Neither event took place in 2015.']], ['23rd National Hockey League All-Star Game', ['The 23rd National Hockey League All-Star Game was held in the St. Louis Arena in St. Louis, home of the St. Louis Blues, on January 20, 1970.', ' It was the first time the All-Star Game was held at the St. Louis Arena.', ' The East Division All-Stars defeated the West Division All-Stars 4–1.', \" Bobby Hull was named the game's most valuable player.\"]], ['53rd National Hockey League All-Star Game', ['The 53rd National Hockey League All-Star Game was held during the 2002–03 NHL season, and took place at the Office Depot Center in Sunrise, Florida, the home of the Florida Panthers, on February 2, 2003.', ' It was the first All-Star Game since the 1997 All-Star Game to use the Eastern Conference–Western Conference format.']], ['52nd National Hockey League All-Star Game', ['The 52nd National Hockey League All-Star Game took place on February 2, 2002, at the Staples Center in Los Angeles, California.', ' The final score was World 8, North America 5.', ' This was the last National Hockey League All-Star Game to have the North America vs. World All-Star format.', ' It was also the last All-Star Game that was held in the same year as the Winter Olympics until the 2018 edition.']], ['58th National Hockey League All-Star Game', [\"The 58th National Hockey League All-Star Game, also known as the 2011 National Hockey League All-Star Game presented by Discover, was the National Hockey League's (NHL) annual All-Star Game played on January 30, 2011.\", ' The game took place during the 2010–11 NHL season at the RBC Center in Raleigh, North Carolina, home of the Carolina Hurricanes.', ' Originally, the Game was supposed to be hosted by the Phoenix Coyotes, but due to ownership issues, the NHL decided to move the game.', ' After bidding for the game reopened, it was awarded to Carolina and fulfilled a nine-year-old promise made to the franchise by NHL Commissioner Gary Bettman.']], ['47th National Hockey League All-Star Game', ['The 47th National Hockey League All-Star Game took place on January 18, 1997, at the San Jose Arena, home of the San Jose Sharks.', ' The final score was Eastern Conference 11, Western Conference 7.', ' This game was originally scheduled for the 1994–95 season, but was cancelled due to the 1994–95 NHL lockout.']], ['6th National Hockey League All-Star Game', ['The Sixth National Hockey League All-Star Game took place at the Detroit Olympia, home of the Detroit Red Wings, on October 5, 1952.', ' For the second year in a row, the format had the First and Second All-Star Teams, with additional players on each team, play each other.', ' After the game ended in a tie for the second year in a row, the NHL decided that they would continue with the previous format of the Stanley Cup winner playing an all-star team.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.531\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adf95745542995534e8c7f8', 'answer': 'in the village of Aldenham', 'question': 'Where does Śivarāma Swami conduct courses on Vaishnava Theology?', 'supporting_facts': [['Sivarama Swami', 3], ['Bhaktivedanta Manor', 0]], 'context': [['Suhotra Swami', ['Suhotra Swami or Suhotra Dasa (born Roger Terrence Crowley, December 11, 1950, Holyoke, Massachusetts – April 8, 2007, Mayapur, India) was a Hindu Vaishnava author, philosopher and a leading guru in the International Society of Krishna Consciousness (ISKCON).', \" He was ISKCON's Governing Body Commissioner (GBC), an initiating spiritual master (diksa guru) and a sannyasi in ISKCON.\", ' He also served as a chairman of the GBC.', ' Since joining ISKCON Suhotra Swami has spent much of his time lecturing and teaching in Europe, especially in Germany and Eastern European countries.', ' Suhotra Swami authored several books on Gaudiya Vaishnava philosophy and Vedanta.']], ['Kenneth R. Valpey', ['Kenneth R. Valpey (born December 18, 1950) is a Gaudiya Vaishnava Theologian who studied at Oxford University, St Cross College (1999–2004).', ' While there, he conducted his research at the Oxford Centre for Hindu Studies.', ' He has a D.Phil.', ' from Oxford University, where his dissertation was on Chaitanya Vaishnava murti-seva.', ' He is also a member of the International Society for Krishna Consciousness, in which Krishna Ksetra Das (spiritual name given to him by Srila Prabhupada) acts as an initiating spiritual master, or guru.', ' He is a professor at Bhaktivedanta College where the central program of study is in Vaishnava Theology.', ' There he teaches courses in Vaishnava Vedanta.', ' He teaches at The Chinese University of Hong Kong and has a Fellowship at the Oxford Centre for Hindu Studies.', ' At present he is working with Ravi M. Gupta on a ‘companion’ to the Bhagavata Purana, and on a translation of a 16th-century Sanskrit Vaisnava ritual texts in corroboration with Dr. Mans Broo (Abo Akademie, Finland).', ' Having taught courses in Indian and Asian religions for the year 2006 at the University of Florida, Gainesville, and having taught for the academic year 2007-08 at the Chinese University of Hong Kong in the Department of Cultural and Religious Studies, he continues to teach at Chinese University of Hong Kong each Autumn semester as a visiting scholar.']], ['Sivarama Swami', ['Śivarāma Swami (born 30 March 1949, Budapest, Hungary) is a Vaishnava guru and a religious leader for the International Society for Krishna Consciousness (ISKCON).', \" He is a member of the Governing Body Commission who is responsible for leading ISKCON's mission in Hungary, Romania and Turkey.\", ' Within ISKCON, Sivarama Swami is also well known for his deep knowledge of Vaishnava literature, and has written several books about Gaudiya Vaishnavism.', ' He has been conducting courses at Bhaktivedanta Manor on his own commentaries to Venu Gita in Gaudiya Vaishnava Theology.']], ['Ravindra Svarupa Dasa', ['Ravindra Svarupa Dasa (born William H. Deadwyler, III) is a religious studies scholar and a Hare Krishna religious leader.', ' He was initiated by A. C. Bhaktivedanta Swami Prabhupada in 1971.', \" He has been a member of ISKCON's Governing Body Commission since 1987, Chairman of that Commission's North American GBC Continental Committee, is the president of ISKCON of Philadelphia, and an ISKCON Guru.\", ' He holds an M.A. and Ph.D. in Religion from Temple University and a B.A. in philosophy from the University of Pennsylvania.', ' He has written extensively on Vaishnava philosophy and used his education to further the discourse of Gaudiya Vaishnava Theology within the context of ISKCON.', ' He is the author of \"Encounter with the Lord of the Universe: Collected Essays 1978-1983\" (Washington, DC: Gita Nagari Press, 1984).', ' He also is featured on Shelter\\'s \"Attaining the Supreme,\" where he gives a lecture on a hidden track.']], ['Bhaktivedanta Manor', ['Bhaktivedanta Manor is a Gaudiya Vaishnava temple set in the Hertfordshire countryside of England, in the village of Aldenham near Watford.', ' The Manor is owned and run by the International Society for Krishna Consciousness (ISKCON), better known as the Hare Krishna movement.', \" It is ISKCON's largest property in the United Kingdom, and one of the most frequently visited Radha Krishna temples in Europe.\", \" It stands in 70 acre of landscaped grounds, consisting of lawns, flower gardens, a children's playground, an artificial lake that attracts many water fowl, and a substantial car park.\"]], ['Govinda Bhashya', ['Govinda Bhashya is a Gaudiya Vaishnava commentary on Vedanta Sutra.', ' It was written in the year 1628 Sakabda (1718 CE) at Galtaji (Galta) near the present city of Jaipur, Rajasthan, by Baladeva Vidyabhushana to defend Gaudiya Vaishnava Theology.']], ['Bhakti Ballabh Tirtha', ['Srila Bhakti Ballabh Tirtha Maharaj (13 April 1924 - 20 April 2017) is a disciple of Bhakti Dayita Madhava Goswami Maharaj and an acharya and initiating spiritual master (Sri Guru) in the Gaudiya Math following the philosophy of the Bhakti marg, specifically of Caitanya Mahaprabhu and Gaudiya Vaishnava theology.', ' He was the President Acharya of Sree Chaitanya Gaudiya Math, headquartered at Kolkata, West Bengal, India and having more than 22 branches in India.', \" He was president of WVA (World Vaishnava Association) and founder of GOKUL( Global organization for KrishnaChaitnaya's Universal Love).\"]], ['Harivamsa Gosvami', ['Harivamsa Gosvami, a disciple of Gopala Bhatta Goswami, espoused a Vaishnava Theology which created the Radhavallabha Vaishnava sect of Hinduism.', ' Also, Harivamsa Goswami is known for his emotional poetry about Radha and Krishna.', ' He was born around 1500 in the village of Bad, in Vrindavan.', ' He was married at the age of 16 and had three sons.', ' He renounced family life at the age of 32 and started for Vrindavana in modern-day Uttar Pradesh.']], ['Bhakti Hridaya Bon', ['Bhakti Hridaya Bon, also known as Swami Bon (Baharpur, 23 March 1901 - Vrindavan, 7 July 1982) was a disciple of Bhaktisiddhanta Sarasvati Thakura and a guru in the Gaudiya Math following the philosophy of the Bhakti marg, specifically of Caitanya Mahaprabhu and Gaudiya Vaishnava theology.', ' At the time of his death, he left behind thousands of Bengali disciples in India.', ' His current successor is Gopananda Bon.']], ['Swami Ramanand', ['Ramanand Swami (born \"Rama Sharma\") to a Brahmin family in Ayodhya in 1738.', ' His parents were Ajay Sharma (father) and Sumati (mother).', ' He was considered to be the incarnation of Uddhava, a close friend of Krishna.', ' Ramanand was the founder and head of the Uddhav Sampraday.', ' Ramanand Swami adopted of the Vishishtadvaita doctrine of the Vaishnava which was first propounded by Ramanuja several centuries earlier.', ' In his travels to Srirangam in southern India in his early life, Ramanand Swami said that Ramanuja gave him diksha (initiation) in a dream and appointed him in his line as an acharya.', ' Ramanand Swami then travelled north to Kathiyavad to spread his philosophy.', ' Before dying in 1802, Ramanand Swami passed the reins of the Uddhav Sampraday to Swaminarayan.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.533\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac1a6bf5542991316484b8d', 'answer': 'California', 'question': 'In what state did this band form that is an American metalcore band, founded in mid-2009, and whose lead vocalist was Jerry Roush?', 'supporting_facts': [['Glass Cloud', 0], ['Glass Cloud', 2], ['Of Mice &amp; Men (band)', 0]], 'context': [['As I Lay Dying discography', ['The discography of American metalcore band As I Lay Dying consists of 6 studio albums, 2 compilation albums, 1 video album, 11 singles and 15 corresponding music videos as well as 1 split album with fellow metalcore band American Tragedy called \"As I Lay Dying/American Tragedy\".']], ['Glass Cloud', ['Glass Cloud was an American metalcore band from Hampton, Virginia formed by Jerry Roush, Joshua Travis, Travis Sykes, and Chad Hasty in late 2011.', ' The band released their debut album, \"The Royal Thousand\" on July 10, 2012 via Equal Vision Records in the United States and Basick Records in the United Kingdom.', ' Lead vocalist Roush was formerly the vocalist of Sky Eats Airplane and Of Mice & Men, while Joshua Travis was the guitarist for The Tony Danza Tapdance Extravaganza until 2012. \"', 'Perfect War Forever\" EP, and was released October 22, 2013 via Equal Vision Records.']], ['St Valentines Massacre (band)', ['St Valentines Massacre is an American metalcore band from Tampa, Florida, formed in 2005.', ' The band members are: Tony Rodriguez the lead vocalist, guitarist/vocalist Paul Zakar, bassist Alan Olshefski, lead guitarist Tyler Jordan, and percussionist Andre Canale.', ' Since their formation, SVM has issued three indie EP releases: \"Of Heroes and Gods\", \"Beneath Crimson Skies\", and \"The Eulogy Sessions\".', ' In January 2007, they won the Rock Solid Pressure industry showcase, as well as placing at Emergenza and other events.', ' St Valentines Massacre played the Ernie Ball stage as part of the 2007 Warped Tour, as well as their national tour in 2009.']], ['Bless the Martyr and Kiss the Child', ['Bless the Martyr and Kiss the Child is the debut album by American metalcore band Norma Jean, released on August 13, 2002 by Solid State Records.', \" This album was the band's only one to feature bassist Joshua Doolittle and vocalist Josh Scogin, the latter of which later became the lead vocalist of The Chariot.\"]], ['Slave to Nothing (song)', ['\"Slave to Nothing\" is a song by American metalcore band Fit for a King.', \" It's the second single from the album Slave to Nothing.\", ' The song has a music video and features Mattie Montgomery, vocalist for metalcore band For Today.']], ['Of Mice &amp; Men (band)', ['Of Mice & Men (often abbreviated OM&M) is an American metalcore band from Orange County, California.', \" The band's lineup currently consists of lead vocalist and bassist Aaron Pauley, lead guitarist Phil Manansala, rhythm guitarist Alan Ashby, and drummer Valentino Arteaga.\", \" The group was founded by Austin Carlile and Jaxin Hall in mid-2009 after Carlile's departure from Attack Attack!\", '.', ' Since 2009, the band has released four studio albums.', ' Carlile departed from the band in December 2016 citing that a long term health condition prompted his exit.', \" After Carlile's departure the band continue to pursue creating music with Pauley taking on both bassist and lead vocalist duties.\"]], ['Sky Eats Airplane (album)', ['Sky Eats Airplane is the second album by American metalcore band Sky Eats Airplane.', ' The album was recorded in early 2008 with producer Brian McTernan and was released on July 22.', ' The album contains 11 tracks, 8 of which are new and 3 of which that are re-recorded versions of previously released demos.', \" The album is the band's first album as a 5 piece and without former vocalist/programmer Brack Cantrell.\", ' It is also the first and last album with vocalist Jerry Roush.', ' Prior to the album\\'s release a video for the first single \"Numbers\" was made and was released on March 27, 2009.']], ['Sleepwalking (Memphis May Fire album)', ['Sleepwalking is the debut full-length studio album by American metalcore band Memphis May Fire.', ' The album was released through Trustkill Records on July 21, 2009.', ' This album is the first album to feature lead vocalist Matty Mullins after former lead vocalist Chase Ryan left the band in 2008.', ' Scheduled to be released summer 2008, the release got delayed because of the line-up change.', ' Mullins has described the album as \"a new breed of rock \\'n\\' roll\" and musically it has \"more aggressive guitar work and noticeably more melodic, but still retains that southern swagger.\"']], ['Dead Throne', ['Dead Throne is the fourth studio album by American metalcore band The Devil Wears Prada.', ' It was released on September 13, 2011 through Ferret Music.', ' Produced by Adam Dutkiewicz, the record was recorded in several studios across the United States from November 2010 – April 2011.', ' The album is a follow up to The Devil Wears Prada\\'s 2009 album, \"With Roots Above and Branches Below\".', \" Much like their previous efforts the album's lyrics were penned by lead vocalist Mike Hranica, whose lyrical direction focused on anti-idolatry concepts.\", ' The band\\'s musical style changed after the success of the \"Zombie EP\", this led to, what Hranica described as; fusing the melodic elements of their previous studio albums with the ferocity of \"Zombie\".']], ['Rise Up Tour', ['The Rise Up Tour is a concert tour co-headlined by American metalcore band The Devil Wears Prada and American metalcore band Memphis May Fire.', ' The tour supports The Devil Wears Prada\\'s Sixth studio album, \"Transit Blues\" and Memphis May Fire\\'s fifth studio album, \"This Light I Hold\".', ' The second leg in Europe added Canadian post-hardcore band Silverstein as a third headliner supporting their eighth studio album \"I Am Alive in Everything I Touch.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.534\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a87b2fc5542996e4f3088d0', 'answer': 'J. R. R. Tolkien', 'question': 'Who wrote the book that inspired the name of the CD/DVD \"Here at the End of All Things\"?', 'supporting_facts': [['Here at the End of All Things', 2], ['The Return of the King', 0]], 'context': [['Week End Tour', ['Week End Tour was the name given to French pop singer Lorie\\'s second concert tour, inspired by her hit single \"Week End\".', ' On 4 October 2004, a live recording of the show was released in two versions, as with the famous singer\\'s previous live effort, \"Live Tour\": a regular jewel case release containing the CD and a DVD and a limited edition cardboard box also containing a poster and a watch featuring the blonde songstress during her performances. On 12 June 2004, a filmed version of the show was released on DVD, once again in two versions.', \" The limited edition DVD additionally contained a bonus disc with additional special features, as well has a new, double-sided poster and a removable tattoo of the singer's name.\"]], ['Living Things +', ['Living Things + is the eleventh DVD of American alternative metal band Linkin Park, originally released on March 22, 2013.', ' The CD version of the video includes all the tracks of the band\\'s fifth studio album, \"Living Things\".', ' The DVD version of the video includes the live version of various songs from all the studio albums by the band, such as \"Hybrid Theory\", \"Meteora\", \"Minutes to Midnight\", \"A Thousand Suns\" and \"Living Things\".', ' The DVD is known as a sequel of the DVD, \"A Thousand Suns+\".']], ['Here at the End of All Things', ['Here At The End Of All Things is a CD/DVD from mathcore band The Number Twelve Looks Like You.', ' It was taped on December 15, 2007 in South Hackensack, New Jersey.', ' The title comes from a quote from the book \"The Lord of the Rings: The Return of the King\".']], ['Un regard 9 Live', ['Un Regard 9 is the title of both the CD and DVD from Lara Fabian\\'s 2005/2006 tour of the same name which followed her 2005 studio release, titled \"9\".', ' The CD and DVD were both released separately and together as a limited edition box set.', ' The CD and DVD were both recorded live on 29 March 2006 at the Zenith in Paris.', ' The CD presents 15 live performances plus a brand new song, \"Aime,\" recorded in a studio in Montreal.', ' This song has been recorded in both English and French, though the latter version is the only one officially released so far.', ' It was initially presented live during a few concerts in Belgium as gift for her native Belgium fans, but the enthusiastic response and feedback was so huge that Lara felt she should record the song and include it on a forthcoming release.']], ['The Return of the King', ['The Return of the King is the third and final volume of J. R. R. Tolkien\\'s \"The Lord of the Rings\", following \"The Fellowship of the Ring\" and \"The Two Towers\".', ' The story begins in the kingdom of Gondor, which is soon to be attacked by the Dark Lord Sauron.']], ['Kollection (film)', ['Kollection is a music documentary about English pop band The Korgis.', ' It was released in 2005 simultaneously with the CD by the same name, \"Kollection\".', ' The acoustic concert especially recorded for this DVD was issued on CD in 2006 as \"Unplugged\".', ' The 2006 re-issue of the DVD also includes video \"Something About The Beatles\".', ' In 2009, the 2006 version of the \"Kollection\" DVD was re-released as a CD/DVD combo under the title \"Something About The Korgis\".']], ['Moonlight Waltz Tour 2011', ['Moonlight Waltz Tour 2011 is a live CD and DVD from the Italian gothic metal band Theatres des Vampires.', ' It is the first all-in-one live CD and DVD release from the band, though Theatres des Vampires have released one live CD (Desire of Damnation) and one live DVD (The Addiction Tour 2006), separately.', ' In addition to the live tracks, the CD contains three new studio songs from The Cult of Lamia soundtrack.', ' The limited edition package also includes a 40-page book.']], ['After... (visual novel)', ['After... is an adult Japanese visual novel developed by Ciel which was released on June 27, 2003 playable on the PC as a CD or a DVD.The early DVD version includes a guide book and soundtrack CD and the early CD version includes a guide book and a mouse pad.', ' Subsequent enhanced ports to the Dreamcast and to the PlayStation 2 as \"After... ~Wasureemu Kizuna~\" were released.', ' Both ports feature their own exclusive characters and new scenarios not in the original PC release.']], ['Love Punch', ['Love Punch is the debut album by Ai Otsuka released on 31 March 2004 under the Avex Trax record label.', ' This album was released in two formats: a CD Only version and a CD+DVD version.', ' The CD+DVD edition comes with a DVD containing promo clips and interviews.', ' Initial pressings contained a limited edition picture book.', ' The album reached #3 on the Oricon charts and stayed on the charts for a total of 98 weeks.', ' Because the album sold 519,300 copies in 2004, it became the #20 most popular album of 2004.', ' However, it also charted on the 2005 end-of-year charts at #94 as it sold 159,025 copies in that year as well.', ' In total, this album has sold 698,277 units.']], ['The Hunting Party (live album)', ['The Hunting Party: Live from Mexico is the twelfth DVD of American alternative metal band Linkin Park, originally released on June 13, 2014.', ' The CD version of the video includes all the tracks of the band\\'s sixth studio album, \"The Hunting Party\".', ' The DVD version of the video includes the live version of various songs from all the studio albums by the band, such as \"Hybrid Theory\", \"Meteora\", \"Minutes to Midnight\", \"A Thousand Suns\" and \"Living Things\".', ' The DVD was available as a special edition content for the promotion of the album.', ' This is the first DVD in the support of an album that does not include any song from the album.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.534\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adf3c155542993a75d2643a', 'answer': 'Anne Perry', 'question': 'What is the given name of the character depicted as Juliet Hulme in Heavenly Creatures?', 'supporting_facts': [['Heavenly Creatures', 0], ['Heavenly Creatures', 2], ['Anne Perry', 0]], 'context': [['Mazengarb Report', ['The Mazengarb Report of 1954, formally titled the \"Report of the Special Committee on Moral Delinquency in Children and Adolescents\", resulted from a ministerial inquiry (the Special Committee on Moral Delinquency in Children and Adolescents) sparked primarily by two infamous and well-publicised events in New Zealand\\'s history: the 22 June 1954 Parker–Hulme murder case (subject of the 1994 Peter Jackson film \"Heavenly Creatures\") and the 20 June 1954 \"Petone incident\".', \" The report gained its name from the inquiry chairman, Queen's Counsel Ossie Mazengarb.\"]], ['India (given name)', ['India is a popular feminine given name derived from the name of the country India, which takes its name from the Indus River.', ' The name was used for India Wilkes, a character in the novel and movie \"Gone with the Wind\".', ' Its use for girls in England began during the British rule in India during the 19th century.', ' It has been used for daughters of aristocratic families in England that had ties to Colonial India, such as India Hicks.', \" Just like names derived from seasons like Summer, Dawn, Solstice, Autumn are feminine, India is internationally recognized as a female name since it's a name of a country and it had been used as a feminine given name for more than hundred years in England and the U.S. Although India is a popular feminine given name, it's not as popular a given name in India as it is around the world.\", ' Girls who are given this name are usually called with a nickname \"indy\", or \"Indie\" which are also popular given names for girls in English speaking countries.']], ['Harp and bowl', ['The Harp and Bowl style of worship, which features musical prayer, derives its name from Revelation 5:8, which describes heavenly creatures which each \"had a harp\" and \"were holding golden bowls full of incense, which are the prayers of the saints.\"']], ['Parker–Hulme murder case', [\"The Parker–Hulme murder case began in the city of Christchurch, New Zealand, on 22 June 1954, when Honora Rieper (also known as Honora Parker, her legal name) was killed by her teenaged daughter, Pauline Parker, and Pauline's close friend Juliet Hulme (later known as Anne Perry).\", ' Parker was 16 at the time, while Hulme was 15.']], ['Heavenly Creatures', ['Heavenly Creatures is a 1994 New Zealand psychological drama directed by Peter Jackson, from a screenplay he co-wrote with his partner, Fran Walsh, about the notorious 1954 Parker–Hulme murder case in Christchurch, New Zealand.', \" The film features Melanie Lynskey and Kate Winslet in their screen debuts with supporting roles by Sarah Peirse, Diana Kent, Clive Merrison, and Simon O'Connor.\", \" The main premise deals with the relationship between two teenage girls, Pauline Parker and Juliet Hulme, who murder Parker's mother.\", ' The events of the film cover the period from their meeting in 1952 to the murder in 1954.']], ['Peter Jackson', ['Sir Peter Robert Jackson {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} (born 31 October 1961) is a New Zealand film director, screenwriter and film producer.', ' He is best known as the director, writer, and producer of \"The Lord of the Rings\" trilogy (2001–03) and \"The Hobbit\" trilogy (2012–14), both of which are adapted from the novels of the same name by J. R. R. Tolkien.', ' Other notable films include the critically lauded drama \"Heavenly Creatures\" (1994), the mockumentary \"Forgotten Silver\" (1995), the horror comedy \"The Frighteners\" (1996), the epic monster remake film \"King Kong\" (2005), and the supernatural drama film \"The Lovely Bones\" (2009).', ' He also produced \"District 9\" (2009), \"The Adventures of Tintin: The Secret of the Unicorn\" (2011), and the documentary \"West of Memphis\" (2012).']], ['Anne Perry', ['Anne Perry (born 28 October 1938 as Juliet Marion Hulme) is an English author of historical detective fiction, best known for her Thomas Pitt and William Monk series.', \" At the age of fifteen she was convicted of participating in the murder of her friend's mother, in 1954.\", ' She changed her name after serving her five-year sentence.']], ['Lisa the Drama Queen', ['\"Lisa the Drama Queen\" is the ninth episode of the twentieth season of \"The Simpsons\".', ' It originally aired on the Fox network in the United States on January 25, 2009 and guest starred Emily Blunt as Juliet.', ' A special version of the end credits theme was performed by Fall Out Boy, although they do not guest star in the actual episode.', ' This was the last episode in the series to be presented only in standard definition, the first regular episode to begin right after the opening credits without a commercial break, with an episode now having four segments, and the last episode to use the original opening sequence starting from Season 2.', ' Brian Kelley returns as a writer after five seasons of being absent from the show.', ' The episode is very loosely based on the 1994 film \"Heavenly Creatures\".']], ['Kate Winslet filmography', ['Kate Winslet is a British actress and singer who has appeared in numerous films and television series.', ' Her film debut was as Juliet Hulme in the 1994 film \"Heavenly Creatures\".', ' She went on to appear in the later 1990s films \"Sense and Sensibility\" as Marianne Dashwood (1995), which earned her a nomination for the Academy Award for Best Supporting Actress, \"Jude\" as Sue Bridehead (1996), \"Hamlet\" as Ophelia (1996) and \"Titanic\" as Rose DeWitt Bukater (1997).', ' For her role in \"Titanic\", it earned her Best Actress nominations for an Academy Award, Golden Globe and Screen Actors Guild Award.']], ['Peter Elliott (actor)', ['Peter Elliott is a leading New Zealand actor.', ' He has appeared in numerous television shows including \"Shortland Street\", \"Gloss\" and .', ' He has also appeared in several movies including \"Heavenly Creatures\".', ' Peter has a daughter Lucy Elliott who is an actress, playing character Dayna Jenkins on \"Shortland Street\" in 2013.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.534\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a807ca5554299485f598613', 'answer': 'the port of Baltimore west to Sandy Hook', 'question': 'Where was the original line of the railroad William Howard worked for?', 'supporting_facts': [['William Howard (engineer)', 0], ['Baltimore and Ohio Railroad', 2]], 'context': [['Rochester, Lockport and Niagara Falls Railroad', ['The Rochester, Lockport and Niagara Falls Railroad(also called the Falls Road Railroad) was a short-lived railroad in Western New York State during the early-1850s.', ' The railroad was incorporated December 14, 1850.', ' This company rebuilt and opened in July 1852, the road originally incorporated April 24, 1834, as the Lockport and Niagara Falls Railroad.', ' The original line was opened in 1838 and sold June 2, 1850.', ' The railroad was consolidated into the New York Central Railroad under the act of 1853.', ' A portion of the line is currently operated as the Falls Road Railroad.']], ['Coudersport and Port Allegany Railroad', ['The Coudersport and Port Allegany Railroad was a short line railroad that operated in Potter and McKean Counties in Pennsylvania in the United States between 1882 and 1964.', ' The original line ran 17 mi along the Allegheny River between the boroughs of Coudersport, the county seat of Potter County, and Port Allegany in McKean County.', ' The line was originally a narrow gauge and converted to in 1889.', ' It was prosperous during a lumber boom in the region and expanded east to Ulysses.', ' The lumber boom ended in the early 20th century and the line slowly declined until 1964, when it was purchased by the Wellsville, Addison and Galeton Railroad.', ' The line was finally abandoned in 1970.', ' Today the only surviving building from the railroad is the Coudersport and Port Allegany Railroad Station, which was listed on the National Register of Historic Places in 1976 and serves as the town hall for Coudersport today.', ' As of 2009, much of the C&PA railroad grading can still be found.', ' Railroad bridge abutments exist at Lillibridge Creek in Port Allegany and along the Allegheny River at Coleman Mills, east of Roulette, and east and west of Coudersport.']], ['William Howard (engineer)', ['William Howard (1793–1834) was an American mechanical engineer who was one of the first to work for the Baltimore and Ohio Railroad.', ' When the railroad built its first cars using friction bearings first developed by Ross Winans, Howard made his own design and patented it on November 2, 1828.']], ['Cane Belt Railroad', ['The Cane Belt Railroad was chartered in the U.S. state of Texas in 1898.', \" Formed by a group of businessmen from Eagle Lake, the short-line railroad was intended to bring the area's sugarcane to market.\", \" In 1902 a disagreement between two of the railroad's chief promoters proved deadly.\", ' By 1904 the line was in operation from Sealy to Matagorda on the Gulf of Mexico.', \" That year the company's stock was bought by the Atchison, Topeka and Santa Fe Railway and the line continued operations under lease to the Gulf, Colorado and Santa Fe Railway starting in 1905.\", ' By the 1920s, the local sugarcane industry collapsed but the railroad was saved by the discovery of two nearby sulphur mines.', ' In 1948, the Cane Belt was merged into the Gulf, Colorado and Santa Fe Railway.', ' In the 1990s most of the original line was abandoned after the last sulphur mine closed.', ' By 2013, only a small portion of the line south of Bay City was operating as part of the BNSF Railway.']], ['Howard C. Whisler', ['Howard C. Whisler (1931–2007) was an American mycologist.', ' Born in Oakland, California, he attended Berkeley schools and then Palo Alto High School.', ' Howard worked on his undergraduate degree at Oregon State College for two years and then went to the University of California, Berkeley, where he completed a Bachelor of Science degree in plant pathology in 1954.', ' He joined the United States Air Force from 1954 to 1956 stationed in Italy.', ' He returned to University of California, Berkeley after his military life and had finished his doctoral degree with Ralph Emerson in 1960.', ' From 1960 to 1961 he held a post doctoral NATO-NSF Fellowship in France, at the Université de Montpellier.', ' Howard was appointed assistant professor of Botany at McGill University in 1961.', ' He was appointed to the faculty at the University of Washington on March 15, 1963 and worked until he died on September 16, 2007, at the age of 76.']], ['Baltimore and Ohio Railroad', ['The Baltimore and Ohio Railroad is the oldest railroad in the United States and the first common carrier railroad.', ' It came into being mostly because the city of Baltimore wanted to compete with the newly constructed Erie Canal (which served New York City) and another canal being proposed by Pennsylvania, which would have connected Philadelphia and Pittsburgh.', ' At first this railroad was located entirely in the state of Maryland with an original line from the port of Baltimore west to Sandy Hook.', ' At this point to continue westward, it had to cross into Virginia (now West Virginia) over the Potomac River, adjacent to the confluence of the Potomac and Shenandoah rivers.', ' From there it passed through Virginia from Harpers Ferry to a point just west of the junction of Patterson Creek and the North Branch Potomac River where it crossed back into Maryland to reach Cumberland.', ' From there it was extended to the Ohio River at Wheeling and a few years later also to Parkersburg, West Virginia.']], ['Art Trouble', ['Art Trouble (1934) is a comedy short starring Harry Gribbon and Shemp Howard.', ' The film is notable for featuring James Stewart in his first screen role.', ' The short was directed by Ralph Staub.', ' Gribbon was one of several comedy team partner with whom Shemp Howard worked.', ' Howard had been an original member of the Three Stooges and brother of Stooges Curly Howard and Moe Howard.', \" Shemp began making his own shorts prior to having to return to the Stooges in the wake of Curly's strokes in the mid-1940s.\"]], ['William Howard Taft IV', ['William Howard Taft IV (born September 13, 1945) is an attorney who has served in the United States government under several Republican administrations.', ' He is the son of William Howard Taft III and the great-grandson of President William Howard Taft.']], ['Joseph Kinsey Howard', ['Joseph Kinsey Howard (February 28, 1906 – August 25, 1951) was an American journalist, historian, and author, who wrote extensively about the history, culture, and economic circumstances of Montana.', ' One of the state\\'s most noted authors of nonfiction, Howard\\'s landmark 1943 book, \"Montana: High, Wide, and Handsome\" is a respected account of Montana history that has influenced later generations of historians.', ' Howard also authored numerous other historic and literary works, and was a vocal, articulate and persuasive advocate for a variety of social, economic and environmental reforms.', ' These endeavors earned Howard the posthumous sobriquet, \"Montana\\'s Conscience.\"', ' Howard believed Montana and the rural West provided the \"last stand against urban technological tedium\" for the individual.', ' He fervently believed that small towns of the sort that predominated in Montana provided a democratic bulwark for society.', \" Howard's writings demonstrate his strong belief in the necessity to identify and preserve a region's cultural heritage.\", ' Howard worked first as a newspaper editor on the \"Great Falls Leader,\" later for the Montana Study (a statewide community development project), and as a freelance writer.', ' His books, speeches and magazine articles, expressed his ideals of community awareness and identity, encouraging readers to retain an idealistic vision contesting the deadening demands of the modern world.']], ['Indiana and Ohio Railway', ['The Indiana and Ohio Railway (reporting mark IORY) is an American railroad that operates 570 mi of track in Ohio, southern Michigan, and parts of southeastern Indiana.', ' It is owned and operated by Genesee & Wyoming, who acquired the railroad in the 2012 purchase of RailAmerica.', ' The Indiana and Ohio Railroad (reporting mark INOH) (merged into the IORY in 1997) was formed in 1978 to operate a branch between Valley Junction, Ohio and Brookville, Indiana.', \" The IORY's original line, acquired in 1985, connected Mason and Monroe, Ohio.\", ' The IORY set up a tourist operation known as the Indiana and Ohio Scenic Railway which operated over this line.', ' The tourist train still operates out of Lebanon, Ohio under the ownership of the Cincinnati Railway Company (CRC) under the name Lebanon Mason Monroe Railroad.', ' Another line, acquired in 1986, runs from Norwood to Brecon, Ohio.', ' In 1991, the former DT&I between Washington Court House, Ohio and Springfield, Ohio came into the system via a designated operator agreement with the West Central Ohio Port Authority.', ' The system expanded north into Michigan in 1997 when it acquired the remainder of the former DT&I mainline between Diann, Michigan and Springfield, Ohio.', ' In 1994, it acquired two lines from Conrail in Springfield, Ohio: one between Springfield and Bellefontaine; and one between Springfield and Mechanicsburg.', ' The Indiana and Ohio Central Railroad (reporting mark IOCR) was the designated owner of these two lines until 2004.', \" One of the I&O's major events took place in 1996 when it was acquired by RailTex.\", ' In 2000 RailTex was absorbed by RailAmerica and in 2004 the I&O absorbed the Indiana and Ohio Central Railroad.', ' Genesee & Wyoming acquired RailAmerica in December 2012.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.536\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adf1b8a5542995534e8c750', 'answer': 'Molly Hatchet', 'question': 'What band did Gator Country guitarist that co-wrote Bloody Reunion come from?', 'supporting_facts': [['Gator Country', 1], ['Duane Roland', 0], ['Duane Roland', 3]], 'context': [['To Sir, with Love (2006 film)', [\"To Sir, with Love () (aka My Teacher, Teacher's Mercy and Bloody Reunion) is a 2006 South Korean horror film, and the feature film debut of director Im Dae-Woong.\"]], ['Monty Byrom', ['Monty Byrom (born July 3, 1958) is an American rock, blues and country guitarist, singer, songwriter and producer.', ' He fronted bands Billy Satellite, New Frontier, and the Academy of Country Music nominated Big House.', \" Earlier in his career Byrom co-produced and co-wrote hit songs for Eddie Money while a member of Money's band.\", ' Money had earlier covered Byrom\\'s Billy Satellite song, \"I Wanna Go Back.\"', ' Later while leading the \"soul country\" band Big House, Byrom made a significant contribution to the new Bakersfield Sound, with a nod to his Bakersfield roots.']], ['Duane Roland', ['Duane Roland (December 3, 1952 – June 19, 2006) was an American guitarist for the Southern hard rock band Molly Hatchet.', ' He was a member of the band from its founding in the mid-1970s until his departure in 1990.', ' During that time he recorded seven albums with the band.', ' He is credited with co-writing some of the band\\'s biggest hits, including \"Bloody Reunion\" and \"Boogie No More\".', ' After leaving the band he played with the Southern Rock Allstars and Gator Country, which included many of the founding members of Molly Hatchet.']], ['WWGR', ['WWGR, commonly called Gator Country 101.9 Music, is a country music radio station based in the Fort Myers, FL area.', ' The station, which is owned by Renda Broadcasting, operates at 101.9 MHz with an ERP of 100 kW.', ' This power gives it one of the best radio signals in all of Southwest Florida.', ' Its transmitter is located off Corkscrew Road in Estero.', ' This incredible signal also has its disadvantages, having exceptional interference with 101.9 \"AMP Radio\" WQMP, an CHR/Top 40 licensed to Daytona Beach, but serves Orlando.', ' The Sebring, Tampa, St. Petersburg, and Avon Park areas are the most affected by this interference with powerful radios receiving those two stations almost as one.']], ['Jimmy Farrar', ['Jimmy Farrar (born 8 December 1950) is a singer, songwriter and musician born in La Grange, Georgia, originally lead singer for the Raw Energy band, Farrar is best known as the second lead singer for the American Southern Rock band Molly Hatchet from 1980 to 1982, and in more recent years, Gator Country.']], ['Bruce Crump', ['Bruce Hull Crump, Jr. (July 17, 1957 – March 16, 2015) was the original drummer with the rock band Molly Hatchet from 1976 to 1982 (including their 1980 hit song \"Flirtin\\' with Disaster\" ) and 1984 to 1991.', ' He also played as a member of the Canadian band Streetheart in the early 1980s, appearing on their \"Live After Dark\" recording, and joined several of his former Molly Hatchet bandmates in the band Gator Country in the mid-2000s.', ' At his death, Crump was in the Jacksonville, Florida-based band White Rhino and the newly reformed China Sky.']], ['WGNE-FM', ['WGNE-FM is commercial radio station that broadcasts to the Jacksonville area on 99.9 FM.', ' The station is licensed to Middleburg to Renda Broadcasting.', ' It is branded as 99.9 Gator Country and broadcasts a country music format.', ' Its studios are in the Arlington district of Jacksonville, and the transmitter is in Downtown Jacksonville.', ' Originally WIYD in Palatka marketed as \"Wide FM\", and sister to WWPF, AM 1260, and formerly \"99.9 Froggy-FM\" Daytona Beach, Florida, the station moved to Jacksonville, Florida in 2005.']], ['Clarence White', ['Clarence White (born Clarence Joseph LeBlanc; June 7, 1944 – July 14, 1973), was an American bluegrass and country guitarist and singer.', ' He is best known as a member of the bluegrass ensemble the Kentucky Colonels and the rock band the Byrds, as well as for being a pioneer of the musical genre of country rock during the late 1960s.', ' White also worked extensively as a session musician, appearing on recordings by the Everly Brothers, Joe Cocker, Ricky Nelson, Pat Boone, the Monkees, Randy Newman, Gene Clark, Linda Ronstadt, Arlo Guthrie, and Jackson Browne amongst others.', ' Together with frequent collaborator Gene Parsons, he invented the B-Bender, a guitar accessory that enables a player to mechanically bend the B-string up a whole tone and emulate the sound of a pedal steel guitar.', ' White was inducted into the International Bluegrass Music Association Hall of Fame in 2016.']], ['Gator Country', ['Gator Country was an American Southern rock band formed in Davie, Florida, in 2005 by several ex-members of the Southern rock group Molly Hatchet.', ' The band, founded by vocalist Jimmy Farrar, guitarist Duane Roland, drummer Bruce Crump, guitarist Steve Holland, and bassist Riff West took its name from the title of the hit song, \"Gator Country\".']], ['WXJZ', ['WXJZ (100.9 FM) is a commercial radio station in Gainesville, Florida, broadcasting to the Gainesville-Ocala, Florida area on 100.9 FM.', ' The station is owned by JVC Media, LLC, through licensee JVC Media of Florida, LLC, and broadcasts a classic hits format billed as \"100.9 WOW FM\".', ' It started on 104.9 MHz before moving to the frequency of the former WYGC \"Gator Country 100.9\".', \" Simultaneously, WYGC moved to WXJZ's 104.9 frequency.\", \" On December 26, 2013, sister station WBXY's Dance format moved over to the 100.9 signal, while JVC simulcasted the 99.5 signal until it was spun off to another company in January 2014.\", ' On September 12, 2015, at Midnight, after stunting throughout the 11th with patriotic music and country songs in a tribute to the 9/11 terrorist attacks, JVC flipped the station to Soft AC and bought the \"Smooth 100.9\" brand back to the market.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.537\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab322b1554299194fa93570', 'answer': 'fictional character', 'question': 'Which type of character is featured by the P.L. Travers\\'s third book and last novel in the \"Mary Poppins\" series?', 'supporting_facts': [['Mary Poppins Opens the Door', 0], ['Mary Poppins (character)', 0]], 'context': [['Saving Mr. Banks', ['Saving Mr. Banks is a 2013 period drama film directed by John Lee Hancock from a screenplay written by Kelly Marcel and Sue Smith.', ' Centered on the development of the 1964 film \"Mary Poppins\", the film stars Emma Thompson as author P. L. Travers and Tom Hanks as filmmaker Walt Disney, with supporting performances by Paul Giamatti, Jason Schwartzman, Bradley Whitford, and Colin Farrell.', ' Deriving its title from the father in Travers\\' story, \"Saving Mr. Banks\" depicts the author\\'s fortnight-long meetings during 1961 in Los Angeles, during which Disney attempts to obtain the screen rights to her novels.']], ['Chim Chim Cher-ee', ['\"Chim Chim Cher-ee\" is a song from \"Mary Poppins\", the 1964 musical motion picture.', ' It was originally sung by Dick Van Dyke and Julie Andrews, and also is featured in the Cameron Mackintosh/Disney \"Mary Poppins\" musical.', ' The song can be heard in the \"Mary Poppins\" scene of The Great Movie Ride at Disney\\'s Hollywood Studios and during the \"Mary Poppins\" segment of \"\" at Disneyland.']], ['Mary Poppins, Goodbye', ['Mary Poppins, Goodbye (Russian: Мэри Поппинс, до свидания!', ' ; translit.', '\\xa0\"Meri Poppins, do svidaniya\") is a Soviet 1983 two-part musical miniseries (part 1 \"Lady Perfection\", part 2 \"Week ends on Wednesday\"), directed by Leonid Kvinikhidze.', ' It is loosely based on Mary Poppins stories by P. L. Travers.', ' The TV series were ordered by the Gosteleradio of USSR and produced by Mosfilm.', ' The official television premiere was on January 8, 1984.']], ['P. L. Travers', ['Pamela Lyndon Travers, OBE ( ; born Helen Lyndon Goff; 9 August 1899 – 23 April 1996) was an Australian-born writer who spent most of her career in England.', ' She is best known for the \"Mary Poppins\" series of children\\'s books, which feature the magical nanny Mary Poppins.']], ['Mary Poppins Opens the Door', ['Mary Poppins Opens the Door is a British children\\'s fantasy novel by the Australian-British writer P.L. Travers, the third book and last novel in the \"Mary Poppins\" series that features the magical English nanny Mary Poppins.', ' It was published in 1943 by Harcourt, Brace & World, Inc and illustrated by Mary Shepard and Agnes Sims.']], ['Mary Poppins (character)', ['Mary Poppins is a fictional character and the eponymous protagonist of P. L. Travers\\' \"Mary Poppins\" books and all of their adaptations.', ' A magical English nanny, she blows in on the East Wind and arrives at the Banks home at Number Seventeen Cherry Tree Lane, London, where she is given charge of the Banks children and teaches them valuable lessons with a magical touch.', ' Travers gives Poppins the accent and vocabulary of a real London nanny: cockney base notes overlaid with a strangled gentility.']], ['Mary Shepard', [\"Mary Eleanor Jessie Knox née Shepard (25 December 1909 – 4 September 2000) was an English illustrator of children's books.\", ' She is best known for the \"Mary Poppins\" stories written by P. L. Travers (1934 to 1988): \"Mary Shepard: Putting Mary Poppins in the picture\", \"The Times\" of London titled an obituary article.', ' She used her married name Mary Knox outside the publishing industry.']], ['Mary Poppins (film)', ['Mary Poppins is a 1964 American musical-fantasy film directed by Robert Stevenson and produced by Walt Disney, with songs written and composed by the Sherman Brothers.', ' The screenplay is by Bill Walsh and Don DaGradi, loosely based on P. L. Travers\\' book series \"Mary Poppins\".', \" The film, which combines live-action and animation, stars Julie Andrews in the role of Mary Poppins who visits a dysfunctional family in London and employs her unique brand of lifestyle to improve the family's dynamic.\", ' Dick Van Dyke, David Tomlinson, and Glynis Johns are featured in supporting roles.', ' The film was shot entirely at the Walt Disney Studios in Burbank, California using painted London background scenes.']], ['Mary Poppins (disambiguation)', ['\"Mary Poppins\" is a series of children\\'s books written by P.L. Travers.']], ['Mary Poppins (musical)', ['Mary Poppins is a musical with music and lyrics by the Sherman Brothers, with additional music and lyrics by George Stiles and Anthony Drewe, and a script by Julian Fellowes.', \" The musical is based on the similarly titled Mary Poppins children's books by P. L. Travers and the 1964 Disney film, and is a fusion of various elements from the two.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.537\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a87081755429960ec39b6fe', 'answer': 'Daredevil', 'question': 'Jessica Jones is a television series created for Netflix, the second in the series of the shows that lead to \"The Defenders\" miniseries after what show released on Netflix earlier?', 'supporting_facts': [['Jessica Jones (TV series)', 0], ['Jessica Jones (TV series)', 1], ['The Defenders (miniseries)', 0]], 'context': [['List of Daredevil characters', ['\"Daredevil\" is an American web television series created for Netflix by Drew Goddard, based on the Marvel Comics character of the same name.', ' It is set in the Marvel Cinematic Universe (MCU), sharing continuity with the films of the franchise, and is the first in a series of shows that will lead up to a Defenders crossover miniseries.', \" The series stars Charlie Cox as Matt Murdock / Daredevil, as well as Deborah Ann Woll, Elden Henson, Rosario Dawson, and Vincent D'Onofrio.\", ' Toby Leonard Moore, Vondie Curtis-Hall, Bob Gunton, and Ayelet Zurer join them for the first season, while Jon Bernthal, Élodie Yung, and Stephen Rider join them for the second.', ' In addition to original characters, several other characters based on various Marvel properties also appear throughout the series.']], ['List of Iron Fist characters', ['\"Iron Fist\" is an American web television series created for Netflix by Scott Buck, based on the Marvel Comics character of the same name.', ' It is set in the Marvel Cinematic Universe (MCU), sharing continuity with the films of the franchise, and is the fourth in a series of shows that will lead up to a Defenders crossover miniseries.', ' The series stars Finn Jones as Danny Rand / Iron Fist, with Jessica Henwick also starring.', ' They are joined by Tom Pelphrey, Jessica Stroup, Ramón Rodríguez, Sacha Dhawan, Rosario Dawson and David Wenham in the first season.', ' In addition to original characters, several other characters based on various Marvel properties also appear throughout the series.']], ['Claire Temple (Marvel Cinematic Universe)', ['Claire Temple is a character portrayed by Rosario Dawson in the television series of the Marvel Cinematic Universe (MCU), based on the comic characters Claire Temple and Night Nurse.', ' A nurse who gives medical aid to vigilantes, she first appeared in the first season of \"Daredevil\" (2015–).', ' Dawson then signed a deal to return for the second season of the series, as well as potentially appear in any other Marvel Netflix series.', ' She has since reprised the role in \"Jessica Jones\", \"Luke Cage\", \"Iron Fist\", and \"The Defenders\".', ' The character has also appeared in a \"Jessica Jones\" tie-in comic.']], ['Daredevil (TV series)', [\"Marvel's Daredevil, or simply Daredevil, is an American web television series created for Netflix by Drew Goddard, based on the Marvel Comics character of the same name.\", ' It is set in the Marvel Cinematic Universe (MCU), sharing continuity with the films of the franchise, and is the first in a series of shows that lead to \"The Defenders\" crossover miniseries.', ' The series is produced by Marvel Television in association with ABC Studios and Goddard Textiles, with DeKnight Productions for the first season.', ' Steven S. DeKnight serves as showrunner on the first season, with Doug Petrie and Marco Ramirez taking over for the second; Goddard serves as a consultant on both seasons.']], ['The Defenders (miniseries)', [\"Marvel's The Defenders, or simply The Defenders, is an American web television miniseries created by Douglas Petrie and Marco Ramirez for Netflix, based on the Marvel Comics characters Daredevil, Jessica Jones, Luke Cage and Iron Fist, who form the eponymous superhero team.\", ' It is set in the Marvel Cinematic Universe (MCU), sharing continuity with the films of the franchise and is the culmination of a series of interconnected shows from Marvel and Netflix.', ' The miniseries is produced by Marvel Television in association with ABC Studios, Nine and a Half Fingers, Inc., and Goddard Textiles, with Ramirez serving as showrunner.']], ['Iron Fist (TV series)', [\"Marvel's Iron Fist, or simply Iron Fist, is an American web television series created for Netflix by Scott Buck, based on the Marvel Comics character of the same name.\", ' It is set in the Marvel Cinematic Universe (MCU), sharing continuity with the films of the franchise and is the fourth in a series of shows that lead to \"The Defenders\" crossover miniseries.', ' The series is produced by Marvel Television in association with ABC Studios, with Devilina Productions and showrunner Buck for the first season.', ' Raven Metzner took over as showrunner for the second season.']], ['Krysten Ritter', ['Krysten Alyce Ritter (born December 16, 1981) is an American actress and former model.', ' Ritter is known for her roles as lead superheroine Jessica Jones on the Marvel Cinematic Universe series \"Jessica Jones\" and \"The Defenders\", Jane Margolis on the AMC drama series \"Breaking Bad\", and Chloe on the ABC comedy series \"Don\\'t Trust the B---- in Apartment 23\".', ' She has appeared in films such as \"What Happens in Vegas\" (2008), \"27 Dresses\" (2008), \"Confessions of a Shopaholic\" (2009), \"She\\'s Out of My League\" (2010), \"Veronica Mars\" (2014), and \"Big Eyes\" (2014).', ' She has also appeared in roles \"Gravity\", \"\\'Til Death\", \"Veronica Mars\", and \"The Blacklist\".']], ['Doug Petrie', ['Douglas \"Doug\" Petrie is an American screenwriter, director, and producer.', ' Best known as a writer, director, and co-executive producer on \"Buffy the Vampire Slayer\".', ' He co-wrote the screenplays for the \"Fantastic Four\" film and \"Harriet the Spy\".', ' He has also written for the television shows \"Angel\", \"The 4400\", \"Tru Calling\" and \"\".', ' He served as a co-executive producer and writer for two seasons on \"\" and as a consulting producer and writer on the second season of \"Pushing Daisies\".', ' He made a cameo on Joss Whedon\\'s web-based film, \"Dr. Horrible\\'s Sing-Along Blog\", as \"Professor Normal\".', ' He served as co-executive producer on the first season of the Netflix show \"Daredevil\", and took over as showrunner for its second season alongside Marco Ramirez.', ' In April 2016 Petrie and Ramirez were announced as showrunners of \"The Defenders\", a miniseries that crosses over \"Daredevil\", \"Jessica Jones\", \"Luke Cage\", and \"Iron Fist\".']], ['Jessica Jones (TV series)', [\"Marvel's Jessica Jones, or simply Jessica Jones, is an American web television series created for Netflix by Melissa Rosenberg, based on the Marvel Comics character of the same name.\", ' It is set in the Marvel Cinematic Universe (MCU), sharing continuity with the films of the franchise, and is the second in a series of shows that lead to \"The Defenders\" crossover miniseries.', ' The series is produced by Marvel Television in association with ABC Studios and Tall Girls Productions, with Rosenberg serving as showrunner.']], ['List of Jessica Jones characters', ['\"Jessica Jones\" is an American web television series created for Netflix by Melissa Rosenberg, based on the Marvel Comics character of the same name.', ' It is set in the Marvel Cinematic Universe (MCU), sharing continuity with the films of the franchise, and is the second in a series of shows that will lead up to a Defenders crossover miniseries.', ' The series stars Krysten Ritter as Jessica Jones, with Rachael Taylor, Eka Darville, and Carrie-Anne Moss also starring.', ' They were joined by Mike Colter, Wil Traval, Erin Moriarty, and David Tennant for the first season, and Leah Gibson and J.R. Ramirez for the second season.', ' In addition to original characters, several characters from other Marvel Cinematic Universe television series and/or based on various Marvel properties also appear throughout the series.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.538\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a80721b554299485f5985ef', 'answer': 'World War II', 'question': 'The Livesey Hal War Memorial commemorates the fallen of which war, that had over 60 million casualties?', 'supporting_facts': [['Livesey Hall War Memorial', 0], ['World War II casualties', 1]], 'context': [['Livesey Hall War Memorial', ['The Livesey Hall War Memorial commemorates the fallen of World War I and World War II who had been employed by the South Suburban Gas Company of London.', ' It is also a tribute to those employees who served in the wars.', ' The monument was designed and executed by British sculptor Sydney March, of the March family of artists.']], ['Alamein Memorial', ['The Alamein Memorial is a Commonwealth War Graves Commission war memorial in the El Alamein War Cemetery, El Alamein, Egypt.', ' The memorial commemorates 11,866 Commonwealth forces members who died during World War II.', ' The memorial was designed by Hubert Worthington and unveiled by Viscount Montgomery of Alamein on 24 October 1954.']], ['Rainham War Memorial', ['The Rainham War Memorial commemorates soldiers killed in both World Wars as well as civilian casualties of World War II.']], ['Portsmouth Naval Memorial', ['The Portsmouth Naval Memorial, sometimes known as Southsea Naval Memorial, is a war memorial in Portsmouth, Hampshire, England, on Southsea Common beside Clarence Esplanade, between Clarence Pier and Southsea Castle.', ' The memorial commemorates approximately 25,000 British and Commonwealth sailors who were lost in the World Wars, around 10,000 sailors in the First World War and 15,000 in the Second World War.', ' The memorial features a central obelisk, with names of the dead on bronze plaques arranged around the memorial according to the year of death.']], ['Freetown Memorial', ['The Freetown Memorial is a Commonwealth War Graves Commission war memorial located outside the Secretariate Building in Freetown, Sierra Leone.', ' The memorial generally commemorates Soldiers from Sierra Leone who died whilst serving with the Royal West African Frontier Force in West Africa and whose graves are not known.', ' The Freetown Memorial commemorates 1,109 First World War casualties and 247 from the Second World War.']], ['Vietnam War Memorial, Hanoi', ['The War Memorial in Hanoi is located across the Ba Dinh Square, across the Ho Chi Minh Mausoleum and close to Hanoi Citadel.', ' Constructed in 1993 in a fusion of traditional Vietnamese and modernist architecture, the memorial commemorates men and women who sacrificed themselves during the Second Indochina War.', ' The war is known by many names, e.g. as the American War in Vietnam.', ' The memorial is a focal point for state functions, commemorating the war dead.']], ['Tower Hill Memorial', ['The Tower Hill Memorial is a Commonwealth War Graves Commission war memorial on the south side of Trinity Square Gardens, in London, England.', ' The memorial commemorates those from the Merchant Navy and fishing fleets who died during both world wars and have \"no grave but the sea\".', ' The memorial was designed by Edwin Lutyens with sculpture work by William Reid Dick, the Second World War extension was designed by Edward Maufe with sculpture work by Charles Wheeler.']], ['Arras Flying Services Memorial', [\"The Arras Flying Services Memorial Commonwealth War Graves Commission war memorial in the Faubourg d'Amiens Cemetery, Arras, France.\", ' The memorial commemorates nearly 1,000 airmen from forces of the Commonwealth who were killed on the Western Front during World War I and who have no known grave.', ' The memorial was designed by Edwin Lutyens, sculpted by William Reid Dick and unveiled by Hugh Trenchard, 1st Viscount Trenchard, Marshal of the Royal Air Force on 31 July 1932.']], ['Masnières Newfoundland Memorial', ['The Masnières Newfoundland Memorial is a Dominion of Newfoundland war memorial that commemorates the actions of the Royal Newfoundland Regiment during the First Battle of Cambrai, of World War I.', ' Located at the north end of the town of Masnières, France, the memorial commemorates the participation of the Newfoundlanders in the taking and defense of the town during the First Battle of Cambrai between the 20th of November and 2 December 1917.']], ['World War II casualties', ['World War II was the deadliest military conflict in history in absolute terms of total casualties.', ' Over 60 million people were killed, which was about 3% of the 1940 world population (est. 2.3 billion).', ' The tables below give a detailed country-by-country count of human losses.', ' World War II fatality statistics vary, with estimates of total deaths ranging from 50 million to more than 80 million.', ' The higher figure of over 80 million includes deaths from war-related disease and famine.', ' Civilians killed totalled 50 to 55 million, including 19 to 28 million from war-related disease and famine.', ' Military deaths from all causes totalled 21 to 25 million, including deaths in captivity of about 5 million prisoners of war.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.539\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a774e9c55429972597f14f3', 'answer': 'Larnelle Harris', 'question': 'David Huntsinger has worked with this gospel singer born in the month of July?', 'supporting_facts': [['David Huntsinger', 4], ['Larnelle Harris', 0]], 'context': [['Karen Clark Sheard', ['Karen Clark-Sheard (born Karen Valencia Clark; November 15, 1960) also known as \"The Maestris Of Gospel\" is an American gospel four-time Grammy Award-winning singer, musician, and songwriter.', ' The youngest daughter of pioneering gospel choral director Mattie Moss Clark, Sheard began her career as a member of the legendary Grammy Award-winning female gospel group The Clark Sisters.', ' She is the mother of contemporary gospel singer and actress Kierra \"Kiki\" Sheard.']], ['David Huntsinger', ['David Huntsinger is a pianist, composer, songwriter, and arranger who moved from his native California to Nashville, TN, in 1976 and played for the Rambos.', ' He co-wrote the song, \"Holy Spirit, Thou Art Welcome\", with Dottie Rambo, as well as the children’s musical, \"Down By The Creek Bank\".', ' In 1979 he left the Rambos to pursue a career as a studio pianist.', ' He wrote and arranged music for the 1989 Grammy-winning album \"A Child’s Gift of Lullabyes\", and arranged for and co-produced Andy Griffith’s 1996 Grammy-winning album, \"I Love To Tell The Story: 25 Timeless Hymns\".', ' He has worked with many artists, such as Sandi Patti, Steve Green, Kathy Troccoli, Michael Crawford, Glen Campbell, Carman, Larnelle Harris, Johnny Cash and Dolly Parton.', ' He toured with Vince Gill for a Christmas tour in 1999, and in 2001 for the Amy Grant/Vince Gill Christmas tour.', ' He has also written a number of children’s musicals, as well as produced many albums of his own original works and arrangements.', ' He played the piano for \"An Unfinished Life\", and contributed to two songs heard in \"The Great Debaters\".', ' He arranged for and played piano in the 1997 Christmas album \"Piano Winterlude\" (Unison).', ' He has done several projects for Discovery House Music.', ' In 2010 he composed a number of pieces for a special 25th Anniversary concert for the international television network 3ABN called \"Pillars of Our Faith\".']], ['Just as I Am (Yolanda Adams album)', ['Just as I Am is the debut album by gospel singer Yolanda Adams, released on September 22, 1987 on the Sound of Gospel label and produced by Thomas Whitfield, who also worked with singer Vanessa Bell Armstrong on early 1980s gospel releases.', ' Though original pressings of the vinyl and cassette release are difficult to find, the album has since been re-released on CD along with another catalog album from Sound of Gospel by Wanda Nero Butler entitled \"New Born Soul\".']], ['Andraé Crouch', ['Andraé Edward Crouch (July 1, 1942\\xa0– January 8, 2015) was an American gospel singer, songwriter, arranger, record producer and pastor.', ' Referred to as \"the father of modern gospel music\" by contemporary Christian and gospel music professionals, Crouch was known for his compositions \"The Blood Will Never Lose Its Power\", \"My Tribute (To God Be the Glory)\" and \"Soon and Very Soon\".', ' In secular music, he was known for his collaborative work during the 1980s and 1990s with Stevie Wonder, Elton John and Quincy Jones as well as conducting choirs that sang on the Michael Jackson hit \"Man in the Mirror\" and Madonna\\'s \"Like a Prayer\".', ' Crouch was noted for his talent of incorporating contemporary secular music styles into the gospel music he grew up with.', ' His efforts in this area helped pave the way for early American contemporary Christian music during the 1960s and 1970s.']], ['J-Shin', ['Jonathan \"J-Shin\" Shinoster is a contemporary R&B/gospel singer born in Liberty City, Florida, who was signed to Slip-n-Slide Records/Atlantic Records and is best known for his hit single \"One Night Stand\" with LaTocha Scott of Xscape.']], ['Larnelle Harris', ['Larnelle Steward Harris (born July 6, 1947) is an American gospel singer and songwriter.', ' During his 30-plus years of ministry, Harris has recorded 18 albums, won five Grammy Awards and 11 Dove Awards, and has had several number one songs on the inspirational music charts.']], ['Kierra Sheard', ['Kierra Valencia \"Kiki\" Sheard (born June 20, 1987 in Detroit, Michigan) is an American gospel singer, fashion designer, and radio host.', ' She is the daughter of gospel singer Karen Clark Sheard (member of gospel singing group The Clark Sisters) and the granddaughter of gospel choral director Mattie Moss Clark.', ' Sheard portrayed Litha in the 2010 Christian-drama film \"Preacher\\'s Kid\".', ' After appearing on her mother\\'s albums, Sheard broke onto the music scene with the release of her debut album \"I Owe You\" in 2004.', ' Her hit single \"You Don\\'t Know\" was written about her mother\\'s bout with an almost fatal blood clot.']], ['Charles Charamba', ['Charles Charamba (born 27 April 1971) is a Zimbabwean gospel musician and a pastor in the Apostolic Faith Mission Church of Zimbabwe.', \" Charamba's albums have been the top selling gospel music in Zimbabwe since the early 2000s.\", ' He has toured both Great Britain and the United States, and throughout southern Africa.', \" One of Charamba's most famous songs is “Machira Chete”.\", ' His wife, Olivia (née Maseko), is also a gospel singer, and appears with him on stage and on his albums; they are often called the \"First Family of Gospel\".']], ['Latice Crawford', ['Latice Crawford (born July 22, 1982 as Latice Tenae Crawford) is an American urban contemporary gospel singer and composer.', ' She is best known for using her three-octave, contralto vocal range of to finish third on the second season of the competitive gospel singing television show \"Sunday Best\" on Black Entertainment Television.', ' Her self-titled debut album reached two \"Billboard\" charts, the Top Gospel Albums and the Top Heatseekers.', ' Her second album, \"Diary of a Church Girl\", also appeared on the Top Gospel Albums chart.']], ['Muyiwa Olarewaju', ['Muyiwa Olarewaju (born 26 November is a British Gospel singer and songwriter who mixes traditional gospel music with elements of world music (from Africa, the Caribbean and Asia), soul, R&B and pop.', ' He is also an experienced performer, broadcaster and presenter.', \" In 2009 he became the first-ever international act to perform on America's popular entertainment channel, BET, for the prestigious annual Celebration of Gospel show.\", ' He has presented the Turning Point programme to an estimated global audience of 70 million.', ' With his group Riversongz, he sold out Indigo2 at the O2 in London, the first gospel act to do so, and he has played at the largest gospel music event in the world, The Experience, attracting a 500,000 strong audience.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.540\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a79c1335542994bb9457066', 'answer': 'author', 'question': 'Eric Ambler and Carol Shields are both best Known for what profession?', 'supporting_facts': [['Eric Ambler', 0], ['Carol Shields', 0]], 'context': [['The New Lot', ['The New Lot is a 1943 British drama film directed by Carol Reed and starring Eric Ambler, Robert Donat, Kathleen Harrison, Bernard Lee, Raymond Huntley, John Laurie, Peter Ustinov and Austin Trevor, with music by Richard Addinsell.', ' The film follows five new recruits from different background and their experiences as they join the army.']], ['The Card (1952 film)', ['The Card is a black-and-white film version of the novel by Arnold Bennett.', ' Entitled The Promoter for its American audience, it was adapted by Eric Ambler and directed by Ronald Neame.', ' It was released in 1952.', ' It starred Alec Guinness as Denry Machin, Petula Clark as Nellie Cotterill, Valerie Hobson as the Countess, and Glynis Johns as Ruth Earp.', ' The film was nominated for the Academy Award for Best Sound.']], ['Eric Ambler', ['Eric Clifford Ambler OBE (28 June 1909 – 22 October 1998) was an influential British author of thrillers, in particular spy novels, who introduced a new realism to the genre.', ' He also worked as a screenwriter.', ' Ambler used the pseudonym Eliot Reed for books co-written with Charles Rodda.']], ['Carol Shields', ['Carol Ann Shields, {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} (née Warner; June 2, 1935 – July 16, 2003) was an American-born Canadian novelist and short story writer.', ' She is best known for her 1993 novel \"The Stone Diaries\", which won the U.S. Pulitzer Prize for Fiction as well as the Governor General\\'s Award in Canada.']], ['W. F. Morris', ['Walter Frederick Morris (31 May 1892–1975) was an English novelist, best known for his mystery novel, \"\" (1929), set in World War I.', ' Critic A.C. Ward praised this as \"an adventure-mystery war-novel with an admirably ingenious and leak-proof plot.', ' This book combines a brilliant exercise of creative imagination with a remarkable ability to reproduce, vividly, first-hand experiences, and there is one brief battle-scene…which is memorable.”', ' (\"The Nineteen-Twenties, Literature and Ideas in the Post-War Decade\", 1930, pp 163–4).', ' Spy novelist Eric Ambler named the book as one of his top five spy stories (in the Afterword to the 1952 edition of his \"Epitaph for a Spy\").']], ['The Cruel Sea (1953 film)', ['The Cruel Sea is a 1953 British war film starring Jack Hawkins, Donald Sinden, Denholm Elliott, Stanley Baker, Liam Redmond, Virginia McKenna and Moira Lister.', ' The film, which was made by Ealing Studios seven years after the end of the Second World War, was directed by Charles Frend and produced by Leslie Norman.', \" It is based on the best selling novel of the same name by former naval officer Nicholas Monsarrat, though the screenplay by Eric Ambler omits some of Monsarrat's grimmest moments.\"]], ['Rough Shoot', ['Rough Shoot, also known as Shoot First, is a 1953 British thriller film starring Joel McCrea and Evelyn Keyes, and featuring Herbert Lom, Marius Goring and Roland Culver.', ' It was directed by Robert Parrish and written by Eric Ambler, based on the 1951 novel \"A Rough Shoot\" by Geoffrey Household.']], ['The Mask of Dimitrios', ['The Mask of Dimitrios is a 1944 American film noir directed by Jean Negulesco and written by Frank Gruber, based on the 1939 novel of the same name written by Eric Ambler (in the United States, it was published as \"A Coffin for Dimitrios\").', ' Ambler is known as a major influence on writers and a developer of the modern thriller genre.']], ['Kevin Shields', ['Kevin Patrick Shields (born 21 May 1963) is an American-born Irish musician, singer-songwriter, composer and producer, best known as the vocalist and guitarist of the alternative rock band My Bloody Valentine.', ' Shields performed in a number of small unsuccessful bands in Dublin, Ireland, as a teenager, before forming My Bloody Valentine with drummer Colm Ó Cíosóig in 1983.', ' Although initially experiencing limited success, the band would later become extremely influential on the evolution of alternative rock with their two original studio albums \"Isn\\'t Anything\" (1988) and \"Loveless\" (1991), both of which pioneered a subgenre known as shoegazing.', ' Shields\\' texturised guitar sound and his experimentation with his guitars\\' tremolo systems resulted in the creation of the \"glide guitar\" technique, which became a recognisable aspect of My Bloody Valentine\\'s sound, along with his meticulous production techniques.']], ['David Coderre', ['David Coderre is the best known advocate of Computer Assisted Audit Tools and Techniques (CAATTs) in the world.', ' CAATTs are computer tools that assist auditors in their profession.', ' In three books, a package of prewritten scripts, and over 20 published articles David Coderre has garnered a reputation in a small but evolving field.', ' David Coderre is best known for his work with Audit Command Language (ACL.)', ' ACL is a computing language designed specifically for the audit profession.', ' Because of his contributions to the field of Internal Audit and CAATTS, David Coderre was awarded a lifetime achievement award by the Canadian body of the Institute of Internal Auditors.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.540\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adcdea85542992c1e3a2441', 'answer': 'Chinese Coffee', 'question': 'What film adaptation do both Jerome Bernard and Ira Lewis have in common?', 'supporting_facts': [['Ira Lewis', 1], ['Ira Lewis', 2], ['Jerry Orbach', 0]], 'context': [['Gerome Ragni', ['Gerome Ragni (born Jerome Bernard Ragni; September 11, 1935 – July 10, 1991) was an American actor, singer and songwriter, best known as the co-author of the groundbreaking 1960s musical \"\".']], ['Les Misérables (1934 film)', [\"Les Misérables is a 1934 film adaptation of Victor Hugo's novel of the same name.\", ' It was written and directed by Raymond Bernard and stars Harry Baur as Jean Valjean and Charles Vanel as Javert.', ' The film lasts four and a half hours and is considered by critics to be the greatest adaptation of the novel, due to its in-depth development of the themes and characters in comparison with most shorter adaptations.']], ['Jerry Orbach', ['Jerome Bernard \"Jerry\" Orbach (October 20, 1935 – December 28, 2004) was an American actor and singer, described at the time of his death as \"one of the last\" bona fide\" leading men of the Broadway musical and global celebrity on television\" and a \"versatile stage and film actor\".']], ['Ira Lewis', ['Ira Lewis Metsky (27 August 1932 — 4 April 2015) was an American actor, writer, and playwright.', ' Lewis was best known for his one-act play, \"Chinese Coffee\", which opened at the Circle in the Square Theatre in 1992, starring Al Pacino.', ' A film adaptation of \"Chinese Coffee\", also starring Pacino, as well as Jerry Orbach, was released in 2000.', \" Ira Lewis wrote the film's screenplay, while Pacino directed the adaptation.\"]], ['Never Gonna Dance', ['Never Gonna Dance is a Broadway musical featuring the music of Jerome Kern.', ' The musical was based on the 1936 Fred Astaire/Ginger Rogers film \"Swing Time\".', ' Lyricists include Oscar Hammerstein, Ira Gershwin, P. G. Wodehouse, Bernard Dougall, Johnny Mercer, Jimmy McHugh, Otto Harbach, and Dorothy Fields.']], ['Pride and Prejudice (1940 film)', ['Pride and Prejudice is a 1940 American film adaptation of Jane Austen\\'s novel \"Pride and Prejudice\", directed by Robert Z. Leonard and starring Greer Garson and Laurence Olivier.', \" The screenplay was written by Aldous Huxley and Jane Murfin, adapted specifically from the stage adaptation by Helen Jerome in addition to Jane Austen's novel.\", ' The film is about five sisters from an English family of landed gentry who must deal with issues of marriage, morality, and misconceptions.', ' The film was released by MGM on July 26, 1940 in the United States, and was critically well received.', ' \"The New York Times\" film critic praised the film as \"the most deliciously pert comedy of old manners, the most crisp and crackling satire in costume that we in this corner can remember ever having seen on the screen.\"']], ['Ira Deutchman', ['Ira Deutchman is best known as a producer, distributor and marketer of independent films, but in 2000, he moved into film exhibition as Co-Founder and Managing Partner of Emerging Pictures—a New York-based digital exhibition company, which was sold in January, 2015 to Vancouver-based 20 Year Media He also served as Chair of the Film Program at Columbia University School of the Arts from 2011 to 2015, where he has been a Professor of Professional Practice for more than 25 years.', ' Deutchman is a member of The Academy of Motion Picture Arts and Sciences.', ' He was one of the original creative advisors to the Sundance Institute and formerly served on the Board of Advisors for the Sundance Film Festival.', ' He has also served as a Board member and former Board chair for the Independent Feature Project, the Board of Advisors for the Los Angeles Independent Film Festival, the Williamstown Film Festival, IFP/West, and the Collective for Living Cinema, and was a member of the Board for Kartemquin Films.', \" In 2015, he donated his personal archives to the University of Michigan's Screen Arts Mavericks and Makers Collection.\", \" Deutchman continues to produce films while consulting on the marketing and distribution of independent films, and teaching producing students in the MFA Film Program at Columbia University's School of the Arts.\", ' Current projects include a film adaptation of Barbara Ehrenreich\\'s best-selling book \"Nickel and Dimed,\" a theatrical adaptation of Joan Micklin Silver\\'s 1976 independent film \"Hester Street\" and a documentary about art film maverick Donald Rugoff, which is in production.', ' He consults for Luce Cinecitta on the marketing of Italian cinema in the United States.', ' Deutchman was awarded the first annual Spotlight Lifetime Achievement Award at the 2017 Sundance Art House Convergence.']], ['Jerome Karabel', ['Jerome Bernard Karabel (born 1950) is an American sociologist, political and social commentator, and Professor of Sociology at the University of California at Berkeley.', ' He has written extensively on American institutions of higher education and on various aspects of social policy and history in the United States, often from a comparative perspective.']], ['Chinese Coffee', ['Chinese Coffee is a one-act play, written by Ira Lewis.']], [\"Rosemary's Baby (miniseries)\", [\"Rosemary's Baby is a 2014 two-part, four-hour television miniseries adaptation of Ira Levin's best-selling horror novel of the same name.\", ' Zoe Saldana stars in the NBC production that is directed by Agnieszka Holland.', ' Unlike earlier versions, it is set in Paris rather than New York.', ' The work was not well received by critics, many of whom said that it was stretched to fill two two-hour timeslots.', ' Although there are several notable changes, this miniseries is considered to be a fairly true updating of the original film adaptation.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.541\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a85fb085542994775f606de', 'answer': 'Ronald Shusett', 'question': 'What is the name of the executive producer of the film that has a score composed by Jerry Goldsmith?', 'supporting_facts': [['Alien (soundtrack)', 0], ['Alien (film)', 2], ['Alien (film)', 5]], 'context': [['Christus Apollo', ['Christus Apollo: Cantata Celebrating the Eighth Day of Creation and the Promise of the Ninth is a cantata in four movements for narrator, mezzo-soprano, choir, and orchestra, based on a text by the science fiction author Ray Bradbury and composed by the American composer Jerry Goldsmith.', ' The piece was commissioned by the California Chamber Symphony in 1969, and premiered later that year in Royce Hall at UCLA with the narration performed by Charlton Heston (who had starred in the 1968 film, \"Planet of the Apes\", for which Goldsmith had composed the musical score).']], ['Alien (soundtrack)', ['The iconic, avant-garde score to the film \"Alien\" was composed by Jerry Goldsmith and is considered by some to be one of his best, most visceral scores.', ' Rather than focusing on themes, Goldsmith creates a bleak and dissonant soundscape that fits the film\\'s dark and intense atmosphere, with only a few \"romantic\" cues.']], ['L.A. Confidential (soundtrack)', ['L.A. Confidential is either the original soundtrack, on the Restless Records label featuring mainly songs and source music, or the original film score, on Varèse Sarabande Records, of the 1997 Academy Award- and Golden Globe Award-winning film \"L.A. Confidential\" starring Kevin Spacey, Russell Crowe, Guy Pearce, James Cromwell, Danny DeVito, and Kim Basinger (who won the Academy Award for Best Supporting Actress for this film).', ' The original score was composed by Jerry Goldsmith.']], ['Innerspace', ['Innerspace is a 1987 American science fiction comedy film directed by Joe Dante and produced by Michael Finnell.', ' Steven Spielberg served as executive producer.', ' The film was inspired by the 1966 science fiction film \"Fantastic Voyage\".', ' It stars Dennis Quaid, Martin Short and Meg Ryan, with Robert Picardo and Kevin McCarthy, with music composed by Jerry Goldsmith.', ' It earned well over $25 million in its domestic gross revenue and won an Oscar, the only film directed by Dante to do so.']], ['Aliens (soundtrack)', ['The score to the 1986 James Cameron film \"Aliens\" was composed by James Horner.', ' The score itself includes musical references to Gayane\\'s Adagio from Aram Khachaturian\\'s \"Gayane\" ballet suite, which had been used in Stanley Kubrick\\'s \"\" (1968).', ' The score also uses musical motifs, sound treatments and excerpts from Jerry Goldsmith\\'s original soundtrack to \"Alien\" (1979).', ' Additional cues taken from Jerry Goldsmith\\'s \"Alien\" score were used in the climax of the film when Horner was unable to finish some cues to Cameron\\'s satisfaction.', \" The film's editors also reportedly altered the score's chronological flow, sometimes looping, truncating or removing the music and placing it in fragmented form in the film out of context.\", ' Despite production issues, it was nominated for an Academy Award in 1986.', ' It was performed by the London Symphony Orchestra.', ' The soundtrack album was released the following year, in 1987.']], ['Mulan (soundtrack)', ['Mulan: An Original Walt Disney Records Soundtrack is the soundtrack for the 1998 Disney animated feature film, \"Mulan\".', ' Released by Walt Disney Records on June 2, 1998, the album featured songs by Matthew Wilder and David Zippel, conducted by Paul Bogaev, and score composed and conducted by Jerry Goldsmith.', ' Vocalists included Lea Salonga, Donny Osmond, 98 Degrees, Jaz Coleman, Stevie Wonder and Christina Aguilera.']], ['Alien (film)', ['Alien is a 1979 science-fiction horror film directed by Ridley Scott, and starring Sigourney Weaver, Tom Skerritt, Veronica Cartwright, Harry Dean Stanton, John Hurt, Ian Holm and Yaphet Kotto.', \" The film's title refers to a highly aggressive extraterrestrial creature that stalks and attacks the crew of a spaceship.\", \" Dan O'Bannon, drawing upon previous works of science fiction and horror, wrote the screenplay from a story he co-authored with Ronald Shusett.\", ' The film was produced by Gordon Carroll, David Giler and Walter Hill through their company Brandywine Productions, and was distributed by 20th Century Fox.', ' Giler and Hill revised and made additions to the script.', ' Shusett was executive producer.', ' The eponymous Alien and its accompanying elements were designed by the Swiss artist H. R. Giger, while concept artists Ron Cobb and Chris Foss designed the more human aspects of the film.']], ['Legend (film score)', ['Legend: The Music of Jerry Goldsmith is a musical film score by American composer Jerry Goldsmith, released in 1986 for the worldwide release of the film of the same name, (excluding the US).', ' The album was released on compact disc in 1992 through Silva Screen records and featured alternate cover art and additional songs.']], ['Total Recall (1990 film)', ['Total Recall is a 1990 American science-fiction action film directed by Paul Verhoeven, starring Arnold Schwarzenegger, Rachel Ticotin, Sharon Stone, Ronny Cox, and Michael Ironside.', ' The film is loosely based on the Philip K. Dick short story \"We Can Remember It for You Wholesale\".', ' It tells the story of a construction worker who is having troubling dreams about Mars and a mysterious woman there.', \" It was written by Ronald Shusett, Dan O'Bannon, Jon Povill, and Gary Goldman, and won a Special Achievement Academy Award for its visual effects.\", ' The original score composed by Jerry Goldsmith won the BMI Film Music Award.']], ['Lionheart (1987 film)', [\"Lionheart, also known as Lionheart: The Children's Crusade, is a 1987 adventure film directed by Franklin J. Schaffner and produced by Talia Shire and Stanley O'Toole.\", \" Shire's brother, Francis Ford Coppola, initially planned to direct the film but instead opted to be executive producer along with Shire's husband, Jack Schwartzman.\", ' The screenplay was written by Menno Meyjes and Richard Outten from a story by Meyjes.', ' The composer Jerry Goldsmith wrote the score.', ' The film was released in August 1987.', ' It was distributed by Orion Pictures.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.542\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac509e95542994611c8b333', 'answer': 'the Twist', 'question': 'The chicken is a type of dance pattern that is a change of pace of what dance?', 'supporting_facts': [['Chicken (dance)', 1], ['Chicken (dance)', 2], ['Dance move', 0]], 'context': [['Dance move', ['Dance moves or dance steps (more complex dance moves are called dance patterns, dance figures, dance movements, or dance variations) are usually isolated, defined, and organized so that beginning dancers can learn and use them independently of each other.', ' However, more complex movements are influenced by musicality and lyrical relevance to express emotions or refer to a message.']], ['Decoded neurofeedback', ['Decoded Neurofeedback (DecNef) is the process of inducing knowledge in a subject by increasing neural activation in predetermined regions of interest in the brain, such as their visual cortex.', ' This is achieved by measuring neural activity in these regions via functional magnetic resonance imaging (FMRI), comparing this to the ideal pattern of neural activation in these regions (for the intended purpose), and giving subjects feedback on how close their current pattern of neural activity is to the ideal pattern.', ' Without explicit knowledge of what they are supposed to be doing or thinking about, over time participants learn to induce this ideal pattern of neural activation.', \" Corresponding to this, their 'knowledge' or way of thinking has been found to change accordingly.\"]], ['Cis AB', ['Cis AB is a rare mutation in the ABO gene which complicates the basic inheritance pattern and blood-transfusion compatibility matching for ABO blood typing.', ' There are different DNA mutations of either type A or Type B alleles that change several amino acids in enzyme transferase A or B, homologous enzymes differing in only four of 354 amino acids (R176G, G235S, L266M, and G268A).', ' A single change in ABO gene DNA could reverse type B to type A and then, a new hybrid enzyme will produce both weak B and A2 (in serum test, A2B and A2B3)).', ' The most common mutation is an A105 allele variation in exon 7 nucleotide position G803C changing Glycine (type A) to Alanine (type B).', ' There are another 8 alleles reported in BGMUT, the most discovered reciently in China and Taiwan.', ' In the cis-AB genotype, both antigens are expressed, like in a standard (trans) AB genotype.', ' In a traditional AB phenotype, A and B antigenes are inherited separately from the father and mother while a cis-AB allele comes from one parent only.', ' In a serum test, cis-AB tests almost the same as a traditional AB, but people with this rare type have problems with blood transfusions.', ' Some of them need components like washed red blood cells or autotransfusion of serum and blood.']], ['Chicken (dance)', ['The Chicken is a popular rhythm and blues dance started in America in the 1950s, in which the dancers flapped their arms and kicked back their feet in an imitation of a chicken.', ' The dance featured lateral body movements.', ' It was used primarily as a change of pace step while doing the Twist.', ' The chicken dance gained even more popularity when Rufus Thomas wrote \"Do the Funky Chicken\", a hit record in 1970.']], ['Bugaku', ['Bugaku (舞楽 , court dance and music) is the Japanese traditional dance that has been performed to select elites mostly in the Japanese imperial court, for over twelve hundred years.', ' In this way, it has been known only to the nobility, although after World War II, the dance was opened to the public and has even toured around the world in 1959.', ' The dance is marked by its slow, precise and regal movements.', ' The dancers wear intricate traditional Buddhist costumes, which usually include equally beautiful masks.', ' The music and dance pattern is often repeated several times.', ' It is performed on a square platform, usually 6 yards by 6 yards.']], ['African-American dance', ['African-American dance has developed within Black American communities in everyday spaces, rather than in studios, schools or companies.', ' These dances are usually centered on folk and social dance practice, though performance dance often supplies complementary aspects to this.', ' Placing great value on improvisation, these dances are characterized by ongoing change and development.', \" There are a number of notable African-American modern dance companies using African-American cultural dance as an inspiration, amongst these are the Whitey's Lindy Hoppers, Alvin Ailey American Dance Theater, Dance Theatre of Harlem, and Lula Washington Dance Theatre.\", ' Unlike European-American dance, African-American dance was not taxed in the fields of Europe where it began and has not been presented in theatrical productions by generations of kings, tzars, and states.', ' Instead, it lost its best dancers to the draft and started requiring taxes from establishments in the form of a federal excise tax on dance halls enacted in 1944.', ' Dance halls continue to be taxed throughout the country while dance studios are not, and African-American dance companies statistically receive less than taxpayer money than European-Americans.', ' However, Hollywood and Broadway have provided wonderful opportunities for African-American artists to share their work and for the public to support them.', ' Michael Jackson and Beyonce are the most well-known African-American dancers.']], ['Dynamic speckle', ['In physics, dynamic speckle is a result of the temporal evolution of a speckle pattern where variations in the scattering elements responsible for the formation of the interference pattern in the static situation produce the changes that are seen in the speckle pattern, where its grains change their intensity (grey level) as well as their shape along time.', ' One easy to observe example is milk: place some milk in a teaspoon and observe the surface in direct sunlight.', ' You will see a \"dancing\" pattern of coloured points.', ' Where the milk dries on the spoon at the edge, the speckle is seen to be static.', ' This is direct evidence of the thermal motion of atoms, which cause the Brownian motion of the colloidal particles in the milk, which in turn results in the dynamic speckle visible to the naked eye.']], ['V6 (Quickstep)', ['V6 is a \"silver\" level dance pattern of the quickstep International Standard Ballroom dance syllabus.', ' The couple moves diagonally to the center (DC) and then diagonally to the wall (DW), thus sweeping a V-shape on the floor.']], ['Frug (dance)', ['The Frug ( or \"froog\") was a dance craze from the mid-1960s, which included vigorous dance to pop music.', ' It evolved from another dance of the era, the Chicken.', ' The Chicken, which featured lateral body movements, was used primarily as a change of pace step while doing the Twist.', ' As young dancers grew more tired they would do less work, moving only their hips while standing in place.', ' They then started making up arm movements for the dance, which prompted the birth of the Swim, the Monkey, the Dog, the Watusi, the Mashed Potato, and the Jerk.', ' The Frug is sometimes referred to as the Surf, Big Bea, and the Thunderbird.']], ['Compulsory dance', ['Compulsory dances, now called pattern dance in ice dancing, are a part of ice dancing and artistic roller skating in which all the couples or solo dancers perform the same standardized steps and holds to music of a specified tempo and genre.', ' One or more compulsory dances were usually skated as the first phase of ice dancing competitions.', ' The 2009-10 season was the final season in which the segment was competed in ISU junior and senior level competition.', ' In June 2010, the International Skating Union replaced the name \"compulsory dance\" with \"pattern dance\" for ice dancing, and merged it into the short dance beginning in the 2010–2011 figure skating season.', ' Compulsory dances are still skated in international roller skating competitions, however as in ice skating, a new section called the Style Dance was introduced from the 2015/16 season alongside the standard compulsory dances and freedance categories.', ' The style dance is very similar in structure to the short dance on ice.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.543\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8b42be55429949d91db515', 'answer': 'novelist', 'question': 'What profession does John Lanchester and Alan Dean Foster have in common?', 'supporting_facts': [['John Lanchester', 0], ['Alan Dean Foster', 0]], 'context': [['Bloodhype', ['Bloodhype (1973) is a science fiction novel by American writer Alan Dean Foster.', ' The book is eleventh chronologically in the Pip and Flinx series, though it was written second; the main characters since they only appear in the last third of the book.', ' Foster originally started the novel as a stand-alone work, but was encouraged by his publishers to include the characters from his previous novel.', ' In the series, it falls after \"Orphan Star\", where Flinx meets the aliens who build him his ship, the Teacher.']], ['Alan Dean Foster', ['Alan Dean Foster (born November 18, 1946) is an American writer of fantasy and science fiction, a prolific creator of and contributor to book series as well as the author of more than 20 standalone novels.', ' He is especially prolific in his novelizations of film scripts.']], ['The Tar-Aiym Krang', ['The Tar-Aiym Krang (1972) is a science fiction novel by American writer Alan Dean Foster.', ' It is Foster’s first published novel and started both his Humanx Commonwealth universe and his two most popular recurring characters, Pip and Philip Lynx (\"Flinx\").', ' The book is second chronologically in the Pip and Flinx series.']], ['The Deluge Drivers', ['The Deluge Drivers (1987) is a science fiction novel by American writer Alan Dean Foster.', ' It is the final entry in Foster\\'s \"Icerigger Trilogy\" of books taking place in the Humanx Commonwealth book series.', ' The two earlier books in the series are \"Icerigger\" and \"Mission to Moulokin.\"']], ['Phylogenesis (novel)', ['Phylogenesis (1999) is a science fiction novel by American writer Alan Dean Foster.', ' It is the first novel in Foster\\'s \"Founding of the Commonwealth Trilogy\".']], ['John Lanchester', ['John Henry Lanchester (born 25 February 1962) is a British journalist and novelist.', \" He was born in Hamburg, brought up in Hong Kong and educated in England; between 1972 and 1980 at Gresham's School in Holt, Norfolk, then at St John's College, Oxford.\", ' He is married to historian and author Miranda Carter, with whom he has two children, and lives in London.']], ['Glory Lane', ['Glory Lane (1987) is a science fiction novel written by Alan Dean Foster.', ' The book takes place outside of either of Foster’s two usual universes, Spellsinger and the Humanx Commonwealth.']], ['Nor Crystal Tears', ['Nor Crystal Tears is a science fiction novel by American writer novel by Alan Dean Foster, first published on 12 August 1982.', \" Foster's ninth book set in the Humanx Commonwealth, it is a first-contact story about the meeting of the insectoid Thranx and Man.\", \" This sets in motion the creation of the Humanx Commonwealth; the political body that is the union of human and thranx society which forms the foundation for many of Foster's science-fiction novels.\"]], ['Sentenced to Prism', ['Sentenced to Prism (1985) is a science fiction novel by American writer Alan Dean Foster, a stand-alone entry in his Humanx Commonwealth series of books.', ' Like many of his books, Foster creates an extraordinary world that he tries to make unlike anything ever seen by his readers by creating a primarily silicon-based planet with almost everything seeming to be made from crystals, glass, and reflective surfaces.']], ['Orphan Star', ['Orphan Star (1977) is a science fiction novel by American writer Alan Dean Foster.', ' The book is Foster\\'s eighteenth published book, his fifth original novel, and is chronologically the third entry in the Pip and Flinx series. \"', 'Bloodhype\" (1973) was the second novel to include Pip and Flinx, but it is eleventh chronologically in the series and the two characters had a relatively small part in that novel\\'s plot.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.543\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab9b2c7554299743d22ebaf', 'answer': 'no', 'question': 'Are both Lygodium or Maxillaria a genus of orchids?', 'supporting_facts': [['Lygodium', 0], ['Maxillaria', 0]], 'context': [['Paracaleana', ['Paracaleana commonly known as duck orchids, is a genus of flowering plants in the orchid family, Orchidaceae that is found in Australia and New Zealand.', ' The Australian species are found in all states but have not been recorded in the Northern Territory.', ' Orchids in this genus are similar to \"Caleana major\", but there are differences in the flowers and in the insects that pollinate them.', ' \"Paraceleana\" orchids, as well as hammer orchids (\"Drakaea\") are pollinated by male thynnid wasps.', ' Duck orchids have a single leaf and one or a few, dull-coloured, inconspicuous flowers.']], ['Heterotaxis', ['Heterotaxis is a genus of orchids native to Latin America from central Mexico to Bolivia, and also to parts of the West Indies.', ' One species extends into Florida (\"H. sessilis\", listed under synonym \"Maxillaria crassifolia\" in Flora of North America).']], ['Ornithidium donaldeedodii', ['Ornithidium donaldeedodii is a species of orchids \"discovered\" in April 2010 when DNA analysis showed that a wrongly labeled orchid at the University of California Botanical Garden in Berkeley, California, was actually a distinct new species.', ' The \"new\" orchid, which had been mislabeled as \"Maxillaria croceorubens\" since the 1990s, was named after orchidologist Donald D. Dod (1912–2008), who collected the specimen in the 1980s in Haiti.', ' The new orchid was officially described in \"Lankesteriana\", an international journal on orchidology, by authors James Ackerman of the University of Puerto Rico and W. Mark Whitten of the Florida Museum of Natural History.']], ['Lygodium', ['Lygodium (climbing fern) is a genus of about 40 species of ferns, native to tropical regions across the world, with a few temperate species in eastern Asia and eastern North America.', ' It is the sole genus in the family Lygodiaceae, though included in the family Schizaeaceae by some botanists.']], ['Microtis (plant)', ['Microtis, commonly known as onion orchids or mignonette orchids is a genus of about 20 species of plants in the orchid family, Orchidaceae.', ' Onion orchids are terrestrial herbs with a single leaf at the base of the plant.', ' They are similar to orchids in the genus \"Prasophyllum\" in that they have an onion-like leaf.', ' The flowers are small but often scented and attractive to their insect pollinators.', ' They are widespread in Asia, Australia and some Pacific islands.']], ['Caladenia', ['Caladenia, commonly known as spider orchids, is a genus of 350 species of plants in the orchid family, Orchidaceae.', ' Spider orchids are terrestrial herbs with a single hairy leaf and a hairy stem.', ' The labellum is fringed or toothed in most species and there are small projections called calli on the labellum.', ' The flowers have adaptations to attract particular species of insects for pollination.', ' The genus is divided into three groups on the basis of flower shape, broadly, spider orchids, zebra orchids and cowslip orchids, although other common names are often used.', ' Although they occur in other countries, most are Australian and 136 species occur in Western Australia, making it the most species-rich orchid genus in that state.']], ['Camaridium', ['Camaridium is a genus of epiphytic orchids widespread across the West Indies and through Latin America from Mexico to Bolivia.', ' One species extends into Florida (\"C. vestitum\", listed in Flora of North America under the synonym, \"Maxillaria parviflora\").']], ['Maxillarieae', ['Maxillarieae is a large and complex tribe of orchids native to South and Central America.', ' Within the tribe there are eight subtribes one of which is that of the genus \"Maxillaria\".']], ['Maxillaria', ['Maxillaria, abbreviated as Max in the horticultural trade, is a large genus of orchids (family Orchidaceae).', ' This is a diverse genus, with very different morphological forms.', ' Their characteristics can vary widely.']], ['Nudol', ['Nudol is a phenanthrenoid of the orchids \"Eulophia nuda\", \"Eria carinata\", \"Eria stricta\" and \"Maxillaria densa\".']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.544\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7b63eb55429931da12ca7e', 'answer': 'New York', 'question': 'What city does Paul Clyne and David Soares have in common?', 'supporting_facts': [['Paul Clyne', 0], ['David Soares', 0]], 'context': [['Rhabdodontidae', ['Rhabdodontids were herbivorous ornithopod dinosaurs from the Cretaceous Period.', ' Rhabdodontids were similar to large, robust \"hypsilophodonts\", with deep skulls and jaws.', ' The family was first proposed by David B. Weishampel and colleagues in 2002.', ' The rhabdodontids were defined as \"the most recent common ancestor of \"Zalmoxes robustus\" and \"Rhabdodon priscus\" and all the descendants of this common ancestor\".', ' In 2005, Paul Sereno defined the family as \"the most inclusive clade containing \"Rhabdodon priscus\" but not \"Parasaurolophus walkeri\"\".', ' The Rhabdodontidae includes the type genus \"Rhabdodon\", \"Zalmoxes\", \"Mochlodon\" and possibly \"Muttaburrasaurus\".', ' Rhabdodontid fossils have been found in Europe and Australia in formations dating from the Early to Late Cretaceous, 100 to 66 million years ago.', ' Ősi \"et al.\" (2012) proposed that \"Rhabdodon\" underwent gigantism on the mainland, as opposed to \"Zalmoxes\" and \"Mochlodon\" experiencing nanism on island habitats.', ' In 2013, Darren Naish discovered a tiny rhabdodont tibia from Sebeș, Romania, turning out to be the smallest known; a paper (Brusette et \"al\" 2013.)', ' later in the same year identified it as \"Zalmoxes\", cf. \"Z. shqiperorum\".']], ['Paul Clyne', ['Paul Clyne was the District Attorney of Albany County, New York from January 2001 through December 2004.', ' A graduate of Albany Law School, he spent about 14 years as an assistant district attorney, before he was tapped by local politicians to replace the retiring District Attorney, Sol Greenberg.', ' He was defeated for re-election by David Soares, first in the Democratic Party primary election in September 2004, and then in the general election in November 2004, in which he ran on an independent line.', ' After a stint teaching at the New York Prosecutors Institute, he went into private practice as a criminal defense attorney in 2007, with an office in Albany, New York.']], ['Turning Point (UK band)', ['Turning Point were a late 1970s fusion band from the UK.', ' The band was formed by Jeff Clyne (bass) and Brian Miller (keyboards), who had previously played together in Isotope, and Pepi Lemer (wordless vocals).', ' Also in the band were David Tidball (saxes), Phil Todd (saxes) and Paul Robinson (drums).', ' They recorded two albums: \"Creatures of the Night\" (1977) and \"Silent Promise\" (1978), both on the Gull label.']], ['Siloam tunnel', ['The Siloam Tunnel (Hebrew: נקבת השילוח\\u200e \\u200e , \"Nikbat HaShiloah\"), also known as Hezekiah\\'s Tunnel, is a water tunnel that was carved underneath the City of David in Jerusalem in ancient times.', ' Its popular name is due to the most common hypothesis of its origin, namely that it dates from the reign of Hezekiah of Judah (late 8th and early 7th century BCE) and corresponds to the Water Works mentioned in in the Bible.', ' According to the Bible, King Hezekiah prepared Jerusalem for an impending siege by the Assyrians, by \"blocking the source of the waters of the upper Gihon, and leading them straight down on the west to the City of David\" ( ).']], ['Common Hope', ['Common Hope is a non-profit organization based out of St. Paul, Minnesota that works with people in Guatemala.', ' Common Hope provides support for over 2,700 children to attend school each year in seventeen villages outside of Antigua and Guatemala City.', ' Common Hope employs a comprehensive approach by focusing their efforts on education, health care, housing, and family development.']], ['David Soares', ['P. David Soares (born October 26, 1969, Brava, Cape Verde) is the Albany County, N.Y. District Attorney.', ' He is a Democrat.']], ['Irmãos Unidos', ['Associação Desportiva, Recreativa e Voluntariado Irmãos Unidos de Vale de Paul, common short form: Irmãos Unidos, uncommon form: Irmãos Unidos da Juventude, Irmãos Unidos de Vale de Paul, abbreative form: ADRV Irmãos Unidos (da Juventude) de Vale de Paul, abbreviative form: ADRV-IUJ (Capeverdean Crioulo, ALUPEC or ALUPEK: \"Irmans Unidus\", \"Irmans Unidos da Juventudi\") is a football club that had played in the Premier division and plays in the Santo Antão North Second Division in Cape Verde.', ' It is based in the city of Pombas, in the eastern part of the island of Santo Antão.']], ['City of David', ['The City of David (Hebrew: (Hebrew: עיר דוד\\u200e , \"Ir David\"; literal translation to Arabic: مدينة داوود\\u200e \\u200e , \"Madina Dawud\", common Arabic name: وادي حلوه , \"Wadi Hilweh\") is an Israeli settlement and the archaeological site which is speculated to compose the original urban core of ancient Jerusalem.', ' First suggested in 1920, the name was used officially from the 1970s, following the capture of East Jerusalem by Israel, but today the name is questioned in the archaeological academic community.', ' In 1997 management of the park was taken over by Ir David Foundation.', ' Although it is located within the Jerusalem Municipality, it is considered a settlement, having been built on land in the West Bank that was occupied by and annexed to Israel following the 1967 Six-Day War and 1980 Jerusalem Law.', ' The international community regards Israeli settlements illegal under international law, although Israel disputes this.', ' It is best known for its Iron Age structures attributed to Judean kings, and it also contains older Canaanite infrastructure dated to the Middle Bronze Age.']], ['Bishopston, Swansea', ['Bishopston (Welsh: Llandeilo Ferwallt ) is a hamlet and community situated on the Gower Peninsula, some 6 mi west south west of the centre of Swansea in South Wales.', ' Forming part of the Bishopston ward of the City and County of Swansea, it is one of the largest villages on Gower.', ' Bishopston has its own rugby club, South Gower RFC, and its own primary and comprehensive schools.', ' Local beaches include Brandy Cove, Caswell Bay and Pwll Du.', ' The electoral ward consists of part or all of the following areas: Barland Common, Caswell, Clyne Common(Rhos-Glyn), Bishopston(Llandeilo-ferwallt), Manselfield(Maesyfaen),Murton(Morthw) Oldway(Henlôn),in the parliamentary constituency of Gower.', ' It is bounded by the Bristol Channel to the south; and the wards of: Pennard to the west; Fairwood(Welsh: Llwynffair) to the north; and Mayals, West Cross(Crwys Fawr) and Newton(Trenewydd)to the east.']], ['Monica Lynn', ['Monica Lynn is an American composer who lives and works in the San Francisco Bay area.', ' She graduated with a Bachelor of Music degree from the University of Texas at Austin, where she studied with Stefan Kostka, David Neumeyer and Forrest Pierce.', ' She continued her education at the University of Missouri in Kansas City where she studied with James Mobberley, Paul Rudy, Chen Yi, Zhou Long, and at the University of California, Santa Cruz where she graduated with a Doctor of Musical Arts degree, studying with Ben Leeds Carson, David Cope, Karlton Hester, David Evan Jones, Hi Kyung Kim, Paul Nauert and Peter Elsea.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.545\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7ea14655429930675135ab', 'answer': 'Oneida Limited', 'question': 'What giant silverware company was started as a religious Utopian group and was for many years run by Pierrepont Noyes?', 'supporting_facts': [['Pierrepont Noyes', 0], ['Pierrepont Noyes', 1], ['Pierrepont Noyes', 2], ['Oneida Community', 6]], 'context': [['Oneida Community', ['The Oneida Community was a Perfectionist religious communal society founded by John Humphrey Noyes in 1848 in Oneida, New York.', ' The community believed that Jesus had already returned in AD 70, making it possible for them to bring about Jesus\\'s millennial kingdom themselves, and be free of sin and perfect in this world, not just in Heaven (a belief called \"Perfectionism\").', ' The Oneida Community practiced communalism (in the sense of communal property and possessions), complex marriage, male sexual continence, and mutual criticism.', ' There were smaller Noyesian communities in Wallingford, Connecticut; Newark, New Jersey; Putney and Cambridge, Vermont.', \" The community's original 87 members grew to 172 by February 1850, 208 by 1852, and 306 by 1878.\", ' The branches were closed in 1854 except for the Wallingford branch, which operated until devastated by a tornado in 1878.', ' The Oneida Community dissolved in 1881, and eventually became the giant silverware company Oneida Limited.']], ['Oneida Community Mansion House', ['The Oneida Community Mansion House is a historic house and museum that was once the home of the Oneida Community, a religiously-based socialist Utopian group led by John Humphrey Noyes.', ' Noyes and his followers moved to the site in Oneida from Putney, Vermont in 1848.', ' The Community lived in the Mansion House communally until 1880, when they dissolved into a joint-stock company.']], ['Margarodidae', ['The Margarodidae or ground pearls (cottony cushion scales, giant coccids, giant scale insects) are a family of scale insects within the superfamily Coccoidea.', ' Members of the family include the Polish cochineal and Armenian cochineal (genus \"Porphyrophora\") and the original ground pearl genus, \"Margarodes\".', ' Beginning in 1880, a number of distinct subfamilies were recognized, with the giant coccis (the Monophlebidae) being the first.', ' Although Maskell proposed a new family, many continued to regard the monophlebids as a mere subfamily for many years, and the Margarodidae classification continued to be polyphyletic through the 20th Century.', ' Since then, taking the advice of Koteja several subfamilies and tribes have been elevated into their own families such as Matsucoccidae and Xylococcidae.', ' The pared-down family of Margarodidae (Margarodidae \"sensu stricto\" or Margarodidae s. s.) is monophyletic.']], ['Birley Spa', ['Birley Spa is a community bath hall and a Victorian bathhouse in the Hackenthorpe district of the City of Sheffield, England.', ' It was built for Charles Herbert Pierrepont, 2nd Earl Manvers and the Lord of the Manor of Beighton in 1842, and initially was a hotel with spa baths beneath.', ' Subsequently, it was used for many years as private dwellings.', ' In the 1973 it was given Grade II listed building status.', ' The building was restored in 2000/2001 and now opens for tours of the bathhouse and grounds.']], ['New Orleans Silversmiths', ['New Orleans Silversmiths is a jewelry and silverware retailer that specializes in both contemporary and antique gold jewelry, as well as antique holloware.', ' It was established in 1938 by Karl Dingeldein, a third generation silversmith from Hanau, Germany who had emigrated to the US.', \" The Dingeldein family's long tradition of metal work and silver manufacture, both in Germany and the U.S., is well documented.\", ' The present owners acquired the business in 1966 and for many years it has been located near the center of the French Quarter, the oldest part of the city.', ' The shop handles new and estate silverware and jewelry.']], [\"Squatter's Cabin\", [\"The Squatter's Cabin is the only remnant of the Kaweah Colony, a socialist utopian group established in the Sierra Nevada in the 1880s.\", ' Now located in Sequoia National Park, the one-room log structure is located at Huckleberry Meadow near the Giant Forest.']], ['Spirit Fruit Society', ['The Spirit Fruit Society was a communitarian group in the United States that was organized after a period of repeated business depressions during the 1890s.', ' The society had its beginnings in Lisbon, Ohio and, over the years of its existence moved to Ingleside, Illinois and, finally, to California.', ' Plagued by rumor, suspicion, and attacks in the press during its early years, the group remained active until 1930.', ' Although it never numbered more than a handful of adherents, the Spirit Fruit Society existed longer and more successfully than any other American utopian group.']], ['The Giant Spider Invasion', [\"The Giant Spider Invasion is a low-budget 1975 science fiction horror film produced by Transcentury Pictures, a partnership owned by the film's director Bill Rebane.\", ' The film is about giant spiders that terrorize the town of Merrill, Wisconsin and the surrounding area.', ' \"The Giant Spider Invasion\" was given a U.S. release in theaters in 1975, and was distributed by Group 1 Films.', ' The iconic theatrical poster art was a throwback to the monster movies of the 1950s.', ' The film received a considerable theatrical run and became one of the fifty top-grossing films of that year.', ' After a three time ABC television network run, the movie achieved additional exposure many years later, when it was featured in a 1997 episode of \"Mystery Science Theater 3000\" (\"MST3K\") (season 8, episode 10).', ' It is now regarded as a cult classic in the B movie realm.', ' The film is listed on \\'The 100 Most Enjoyably Bad Movies Ever Made\\' in the book \"The Official Razzie Movie Guide\" by Golden Raspberry Award founder John Wilson.']], ['Qarmatians', ['The Qarmatians (Arabic: قرامطة\\u200e \\u200e \"Qarāmita\"; also transliterated Carmathians, Qarmathians, Karmathians) were a syncretic religious group that combined elements of Zoroastrianism with the Ismaili Shia Islam centered in al-Hasa (Eastern Arabia), where they established a religious utopian republic in 899 CE.', ' They are most famed for their revolt against the Abbasid Caliphate.', ' Mecca was sacked by the sect’s leader, Abu Tahir al-Jannabi, outraging the Muslim world, particularly with their theft of the Black Stone and desecration of the Zamzam Well with corpses during the Hajj season of 930 CE.']], ['Pierrepont Noyes', ['Pierrepont Burt Noyes (August 18, 1870 – April 15, 1959) was an American businessman and writer.', ' He was brought up in the Oneida Community, a religious Utopian group.', ' Noyes later became the head of Oneida Limited, a position he held for many years.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.546\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a80b4d1554299260e20a134', 'answer': 'no', 'question': 'Are Harry Kizirian and Howard Kazanjian known for the same industry?', 'supporting_facts': [['Howard Kazanjian', 0], ['Howard Kazanjian', 1], ['Harry Kizirian', 0]], 'context': [[\"Harry's Harbour, Newfoundland and Labrador\", [\"Harry's Harbour is a village located southwest of Baie Verte, Newfoundland and Labrador, Canada.\", ' The Way office was established in 1889.', ' The first Waymaster was Robert Howard.', ' It had a population of 199 in 1940 and 192 in 1956.', \" Harry's Harbour boasts a beautiful bay and an underground fishing industry.\", ' Freake is a popular last name for the area.', ' Harry’s Harbour was not in the General Census of Newfoundland for the years 1836, 1845 or 1857.', ' It appeared for the first time in 1874, in the District of Twillingate – #65.', ' It is assumed that the previous communities of Salmon Cove and Jerry’s Cove became the community of Harry’s Harbour.']], ['Pattie Howard', ['Pattie Howard (born Francine Patience Howard, April 12, Cleveland, Ohio, United States) is an American Gospel and R&B Singer-Songwriter, Producer, Composer and Vocal Arranger.', ' She is a music industry veteran who has released two albums with major record labels, RCA Records and Light Records.', ' Howard, who owns her own full service entertainment company, PH Balanced Music, is also known for singing background for many major artists including Whitney Houston, Gladys Knight, Brandy, Mary J Blige, Fantasia, Queen LaTifah, Madonna, Andrae Crouch, Michael Jackson, Curtis Siger, Lisa Stanfield, BeBe and CeCe Winans, Reba Rambo,and Diana Ross.', ' She has dozens of gold and platinum albums to her credits encompassing artists from almost every genre.', ' Howard landed one of her most profound gigs traveling the world with Whitney Houstonfrom 1992-2001, at the height of her career, The Bodyguard Era.', ' During the early 2000s, Howard returned to songwriting, music production, mixing, and mastering and is currently singing (solo & background), composing, arranging and producing various artists/bands.', ' In 2016 Pattie released 2 singles through her record label PH Balanced Music.', ' \"Jesus Is His Name\" introduces Pattie\\'s daughter Shekinah Nicole Howard in a contemporary gospel duet produced by Wow Jones and co produced by Pattie Howard.', ' The second single titled \"Feel Me, Heal Me\" was also Produced by Wow Jones, written and arranged by Pattie Howard.']], ['Raiders of the Lost Ark', ['Raiders of the Lost Ark (also known as Indiana Jones and the Raiders of the Lost Ark) is a 1981 American action adventure film directed by Steven Spielberg, with a screenplay written by Lawrence Kasdan, from a story by George Lucas and Philip Kaufman.', ' It was produced by Frank Marshall for Lucasfilm Ltd., with Lucas and Howard Kazanjian as executive producers.', ' Starring Harrison Ford, it was the first installment in the \"Indiana Jones\" film franchise to be released, though it is the second in internal chronological order.', ' It pits Indiana Jones (Ford) against a group of Nazis who are searching for the Ark of the Covenant, which Adolf Hitler believes will make his army invincible.', \" The film co-stars Karen Allen as Indiana's former lover, Marion Ravenwood; Paul Freeman as Indiana's rival, French archaeologist René Belloq; John Rhys-Davies as Indiana's sidekick, Sallah; Ronald Lacey as Gestapo agent Arnold Toht; and Denholm Elliott as Indiana's colleague, Marcus Brody.\"]], ['Return of the Jedi', ['Return of the Jedi (also known as Star Wars: Episode VI – Return of the Jedi) is a 1983 American epic space opera film directed by Richard Marquand.', ' The screenplay by Lawrence Kasdan and George Lucas was from a story by Lucas, who was also the executive producer.', ' It was the third installment in the original \"Star Wars\" trilogy and the first film to use THX technology.', ' The film is set one year after \"The Empire Strikes Back\" and was produced by Howard Kazanjian for Lucasfilm Ltd.', ' The film stars Mark Hamill, Harrison Ford, Carrie Fisher, Billy Dee Williams, Anthony Daniels, David Prowse, Kenny Baker, Peter Mayhew and Frank Oz.']], ['Steve Chapin', ['Stephen Chapin (born 30 Dec 1946) is an American singer/songwriter.', ' He is best known as the youngest of the four Chapin brothers, which include Harry Chapin and Tom Chapin and is son of drummer Jim Chapin and Elspeth Burke Chapin Hart, editor, artist and matriarch of the Burke, Leacock, Chapin clan.', ' He is the father of Christina Chapin, Frankie Chapin, and Jonathan Chapin.', ' He is the uncle of Jen Chapin and The Chapin Sisters.', ' He has toured nationally and Internationally, with his own band; The Harry Chapin Band; and with his late brother Harry Chapin as his band leader, musical director, arranger, producer, piano player/multi instrumentalist and singer.', \" He continues to perform concerts all over the world with his band The Harry Chapin Band which includes the original members of the band: Steve Chapin, Big John Wallace, and Howard Fields, and new members since 2005, Clark Wallace (Big John's son) and Jonathan Chapin (Steve's son).\", ' Steve has also produced many albums including \"The Chapin Family Christmas album\", and has worked as a teacher, arranger, recording artist, commercial producer, performer, singer, and songwriter.', ' He has appeared on all Harry Chapin albums, and arranged and produced most of them.', ' Most notably, Harry Chapin\\'s \"Greatest Stories Live\" album which includes his performance of one of his songs \"Let Time Go Lightly\".']], ['Apple Capital Museum', ['The Apple Capital Museum is a museum located in Berwick, Nova Scotia exploring the history of the Town of Berwick and near-by Kings County communities.', ' The museum is housed in a restored 19th century store, originally the Harry Lyons harness shop.', \" In the 1940s, it was purchased by Howard Margeson who operated a men's clothing store, taxi business and bicycle shop.\", ' It was donated to the Museum in 1998 by the Margeson family.', ' The Museum was founded in 1998 and shares the building with the tourist bureau for the Town of Berwick.', \" The apple industry is a major focus and the Museum includes a large working railway model of the town’s centre during the height of Nova Scotia's apple industry in the 1930s with the extensive tracks and sidings of the Dominion Atlantic Railway.\", ' The Museum is run by the Apple Capital Museum Society and is open seasonally.']], ['Chris Enss', ['Chris Enss (born 1961) is an American author and screenwriter.', ' Enss has written more than 20 books on the subject of women in the Old West, and has collaborated with producer Howard Kazanjian on four books, including two about Roy Rogers and Dale Evans.']], ['Conjuring (book)', ['Conjuring is an illustrated book by James Randi.', \" Randi gives a detailed history of conjuring, more commonly known as magic, said to be the world's second oldest profession.\", ' It includes detailed portraits of conjurors, including the Harry Blackstone, Sr., Harry Blackstone, Jr., Harry Houdini and his entourage, Howard Thurston, Robert Heller, Joe Berg, and others.']], ['Harry Kizirian', ['Harry Kizirian (Armenian: Հէրի Գիզիրեան ; July 13, 1925 – September 13, 2002) was an Armenian American member of the United States Marine Corps who served during World War II.', \" Kizirian's service lasted from February 1944 to February 1946, during which he spent seventeen months overseas.\", ' Kizirian took part in the Battle of Okinawa, where he landed during the first assault wave while heading a Marine fire team.']], ['Howard Kazanjian', ['Howard G. Kazanjian (born July 26, 1942) is an American film producer known for \"Raiders of the Lost Ark\" and \"Return of the Jedi\".', ' Kazanjian is also a former 8-year Vice President of Lucasfilm, Ltd., and a published non-fiction author.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.546\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae537e45542990ba0bbb235', 'answer': 'October 13, 1980', 'question': \"When was the American singer, songwriter, record producer, dancer and actress born who's second studio album is Chapter II?\", 'supporting_facts': [['Chapter II (Ashanti album)', 0], ['Ashanti (singer)', 0]], 'context': [['Chapter II (Benga album)', ['Chapter II is the third studio album by British record producer Benga.', ' The album was released on 6 May 2013 through Sony Music.', ' It entered the UK Albums Chart at number 93.']], ['Like a Virgin (album)', ['Like a Virgin is the second studio album by American singer and songwriter Madonna, released on November 12, 1984, by Sire Records.', ' Following the success of her self-titled debut album, Madonna wanted to become the record producer of her next album.', ' However, her label was not ready to give her the artistic freedom and she chose Nile Rodgers instead to produce the album due to his work with David Bowie.', ' Madonna wrote six songs on the record, five of which feature Steve Bray as a co-writer.', ' The album was recorded at Power Station Studio in New York at a quick pace.', ' Rodgers enlisted the help of his former Chic bandmates Bernard Edwards, who was the bassist, and Tony Thompson, who played drums.', \" Jason Corsaro, the record's audio engineer, persuaded Rodgers to use digital recording, a new technique introduced at that time.\"]], ['Capítulo II: Brinca', ['Capítulo II: Brinca (English: \"Chapter II: Jump\") is the second studio album by Mexican-American recording artist DJ Kane.', ' It was released on September 13, 2005 by EMI Latin.']], ['Rock wit U (Awww Baby)', ['\"Rock wit U (Awww Baby)\" is a song by American R&B singer Ashanti.', ' It was released in May 2003 as the lead single from her second studio album, \"Chapter II\".', ' The song peaked at number two on the U.S. \"Billboard\" Hot 100 and became her first international hit from her second album, reaching number seven in the UK and number 19 in Australia.', ' The song was certified Gold by the RIAA on June 16, 2010.']], ['Anthony Hamilton (musician)', ['Anthony Cornelius Hamilton (born January 28, 1971) is an American singer, songwriter, and record producer who rose to fame with his platinum-selling second studio album \"Comin\\' from Where I\\'m From\" (2003), which featured the title track single \"Comin\\' from Where I\\'m From\" and the follow-up \"Charlene\".', ' Nominated for 10 Grammy Awards, he is also known for the song \"Freedom\" from the soundtrack album of \"Django Unchained\" co-written and sung as a duo with indie soul singer Elayna Boynton.']], ['Ashanti (singer)', ['Ashanti Shequoiya Douglas (born October 13, 1980), known simply as Ashanti, is an American singer, songwriter, record producer, dancer and actress.', ' Ashanti is known for her eponymous debut album, which featured the hit song \"Foolish\", and sold over 505,000 copies throughout the U.S. in its first week of release in April 2002.', ' In 2003, the album won Ashanti her first Grammy Award for Best Contemporary R&B album.', ' Her second release achieved Platinum status and other top 10 singles.']], ['Chapter II (Ashanti album)', ['Chapter II is the second studio album by American R&B singer Ashanti, released by Murder Inc. and Island Def Jam on July 1, 2003 in the United States.', ' The album involves production by Irv Gotti and Chink Santana and features a guest appearance by Ja Rule.', ' \"Chapter II\" spawned three singles including the \"Billboard\" Hot 100 number two song \"Rock Wit U (Awww Baby)\", the R&B-ballad \"Rain on Me\" and the Irv Gotti-produced \"Breakup 2 Makeup (Remix)\".']], ['Guillotine IV (The Final Chapter)', ['\\'Guillotine IV (The Final Chapter)\\' is the second single from Falling in Reverse\\'s third album \"Just Like You\".', ' It is the fourth and final installment of the Guillotine series, which was started by Escape the Fate when former lead singer Ronnie Radke was in the band.', \" The first song titled 'The Guillotine' was in Escape the Fate's debut album Dying Is Your Latest Fashion in 2006.\", \" The second song which was titled 'This War Is Ours (The Guillotine II)' was on Escape the Fate's second studio album This War Is Ours in 2008.\", \" The third song which was titled 'The Aftermath (The Guillotine III)' was on Escape the Fate's third self-titled album in 2010.\", ' Then finally in 2015 Falling in Reverse finished the series with \"\"Guillotine IV (The Final Chapter).']], ['Chink Santana', ['Andre Parker, known by his stage name Chink Santana, is an American R&B musician and producer from Washington, D.C. His achievements include co-producing \"Ashanti\", the award-winning debut album from the singer of the same name.', ' He also worked on her follow-up, \"Chapter II,\" and has recently co-written and produced tracks on \"Judgement Days,\" the second album by English singer/rapper Ms. Dynamite.', ' He was also part of the making of Jim Jones\\' album, \"Hustler\\'s P.O.M.E. (Product of My Environment)\".']], ['FutureSex/LoveSounds', ['FutureSex/LoveSounds is the second studio album by American singer and songwriter Justin Timberlake.', ' It was released on September 8, 2006 by Jive Records and its affiliated label of the Zomba Group of Companies.', \" During a two-year hiatus, Timberlake resolved his feelings on being unable to record any new material, and as he returned to record some new music, he began collaborating with his longtime record producer Timbaland, alongside the latter's colleague Danja.\", \" The album's contents were produced at Timbaland's Thomas Crown Studios.\", ' The album shares some lyrical themes with Timberlake\\'s debut album \"Justified\" (2002), although \"FutureSex/LoveSounds\" has more diversity in music genres.', ' It infuses R&B and pop with techno, funk, and elements of rock.', \" The reprises and interludes interspersed on the album's tracks were created by the production team with the goal of channeling Timberlake's influences; including David Bowie and Prince.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.547\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a80b3a9554299485f5986cc', 'answer': 'Fairfax County', 'question': 'Tysons Galleria is located in what county?', 'supporting_facts': [['McLean, Virginia', 0], ['Tysons Galleria', 0]], 'context': [[\"Scott's Run Nature Preserve\", [\"Scott's Run Nature Preserve is a nature preserve in Fairfax County, Virginia, United States.\", ' Located in McLean, it is bordered by Virginia State Route 193 to its south, Interstate 495 to its east and the Potomac River to its north.', \" It encompasses 336 acre of woodland with its namesake, Scott's Run, flowing through its west side.\", \" Scott's Run originates in nearby Tysons Corner and enters the Potomac on the northwest side of the preserve.\", ' The preserve is noted for including eastern hemlocks among its plant life, which are rare for the area.', ' It is a popular destination for recreation and hiking and is operated by the Fairfax County Park Authority.']], ['Tysons Galleria', ['Tysons Galleria is a three-level super-regional mall owned by General Growth Properties located at 2001 International Drive, McLean, Virginia, in Tysons Corner.', ' It is the second-largest mall in McLean/Tysons Corner, and one of the largest in the Washington metropolitan area.']], ['Poughkeepsie Galleria', ['The Poughkeepsie Galleria (locally known as \"The Galleria\") is an upscale shopping center on U.S. 9 in the Town of Poughkeepsie, New York, located just north of Wappingers Falls, and is the largest shopping center in Dutchess County.', ' The Galleria has an area of 1100000 sqft with two floors containing 140 shops and 14 restaurants as well as a 16-screen, stadium-seating Regal Cinemas theater.']], ['Tysons, Virginia', ['Tysons, or formerly “Tysons Corner” is a census-designated place (CDP) and unincorporated community in Fairfax County, Virginia, United States.', ' Located in Northern Virginia between the community of McLean and the town of Vienna along the Capital Beltway (I-495), it lies within the Washington Metropolitan Area.', ' Tysons is home to two super-regional shopping malls—Tysons Corner Center and Tysons Galleria—and the corporate headquarters of numerous companies such as Intelsat, Gannett, Hilton Worldwide, Freddie Mac, Capital One and Booz Allen Hamilton.', \" Tysons is Fairfax County's central business district and a regional commercial center.\", ' It has been characterized as a quintessential example of an edge city.', ' The population was 19,627 as of the 2010 census.']], ['Tysons Corner Center', ['Tysons Corner Center, located in the Tysons Corner unincorporated area in Fairfax County, Virginia, United States (between McLean and Vienna, Virginia), opened to the public in 1968, becoming one of the first fully enclosed, climate-controlled shopping malls in the Washington metropolitan area.']], ['Cobb Galleria Centre', ['The Cobb Galleria Centre is a meeting and convention center and a shopping center in the Cumberland/Galleria district of Cobb County, northwest of Atlanta, Georgia, in the United States.', ' It is also located next to a cluster of mid-rise office buildings, the Cumberland Mall and the Cobb Energy Performing Arts Centre.', ' It has hosted over 15,000 events and millions of guests.', ' The venue operates under the direction of the Cobb-Marietta Coliseum and Exhibit Hall Authority and is located at the intersection of three major highways: Interstate 75, Interstate 285, and Cobb Parkway (U.S. 41) just northwest of the city.', ' The Galleria Specialty Mall, which pre-dates the convention center, is located downstairs, with meeting halls upstairs.']], ['West McLean, Virginia', ['West McLean is an unincorporated community in Fairfax County, Virginia, United States.', ' West McLean is located in the western part of the McLean census-designated place and includes much of the Tysons Corner area.', ' West McLean has its own post office which has ZIP code 22103, which is used primarily for the PO Boxes at that Post Office.', ' Other than the Post Office itself, West McLean uses ZIP code 22102.']], ['McLean station', ['McLean (preliminary name Tysons East, Tysons–McLean) is a Washington Metro station in Fairfax County, Virginia, on the Silver Line.', ' The station is located in Tysons Corner, with a McLean postal address.', ' It began operation on July 26, 2014.']], ['McLean, Virginia', ['McLean ( ) is a census-designated place (CDP) in Fairfax County in Northern Virginia.', ' McLean is home to many diplomats, businessmen, members of Congress, and high-ranking government officials partially due to its proximity to Washington, D.C. and the Central Intelligence Agency.', ' It is the location of Hickory Hill, the former home of Ethel Kennedy, the widow of Robert F. Kennedy.', ' It is also the location of Salona, the former home of Light-Horse Harry Lee, the Revolutionary War hero.', ' The community had an estimated total population of 53,673 in 2015, according to estimates prepared by the United States Census Bureau.', ' It is located between the Potomac River and the town of Vienna.', ' McLean is known for its luxury homes and its high-end shopping destinations: the Tysons Corner Center and the Tysons Galleria.', ' The two McLean zip codes - 22101 and 22102 - are among the most expensive ZIP Codes in Virginia and the United States.']], ['Spring Hill station', ['Spring Hill (preliminary names Tysons West, Tysons–Spring Hill Road) is a Washington Metro station in Fairfax County, Virginia, on the Silver Line.', ' Located in Tysons Corner, it began operation on July 26, 2014.', ' The station is located in the central median of Leesburg Pike (SR 7) just west of Spring Hill Road.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.548\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab531cc5542990594ba9d23', 'answer': '728,000 ft²', 'question': 'How large is the shopping mall where KGOT radio station has its studios ?', 'supporting_facts': [['KGOT', 1], ['Dimond Center', 0]], 'context': [['Bedok Mall', ['Bedok Mall (Chinese: 勿落广场) is a large suburban shopping mall in Bedok, Singapore and part of a mixed development compromising of retail and residential development that is integrated with a bus interchange.', ' Located at the heart of Bedok, the integrated retail and residential development comprises a 3-storey lifestyle and family shopping mall and eight 15-storey residential towers called Bedok Residences.', ' It was the first major shopping mall to open in Bedok and was built on the site of the former Bedok bus interchange.']], ['Mall of Georgia', ['Mall of Georgia is an enclosed super-regional shopping mall located in Gwinnett County, Georgia, near the city of Buford, 30 mi northeast of Atlanta.', ' Built in 1999, it is currently the largest shopping mall in the state of Georgia, consisting of more than two hundred stores on three levels.', \" The mall's anchor stores include Belk, Dillard's, JCPenney, Macy's and Von Maur, other major stores include Barnes & Noble, Dick's Sporting Goods, and Haverty's.\", ' Also, located in the Mall of Georgia Crossing is Best Buy, Nordstrom Rack, T.J. Maxx, and Target.', ' Also featured in the mall is a large village section, comprising lifestyle tenants and restaurants in an outdoor setting, as well as a 500-seat amphitheater.', ' The mall attracts many high end stores such as Coach, Swarovski, Clarks, J.Crew, and Aveda.', ' Simon Property Group manages the Mall of Georgia.']], ['Kentucky Oaks Mall', ['Kentucky Oaks Mall is an enclosed super-regional shopping mall in Paducah, Kentucky, USA.', ' Managed by Cafaro Company, the mall includes more than 90 inline stores, as well as regional radio station Rock 98.3 WJLI.', \" Its anchor stores comprise JCPenney, Best Buy, Elder-Beerman, a Dillard's store divided into two sub-stores, and Dick's Sporting Goods.\", \" It was the largest mall in Kentucky by gross leasable area when it opened, and remains the state's third-largest, behind Fayette Mall in Lexington and Mall St. Matthews in Louisville.\"]], ['Mall of New Hampshire', ['The Mall of New Hampshire is a shopping mall located in the Lower South Willow neighborhood of Manchester, New Hampshire.', \" Its major anchoring stores are Macy's, Old Navy, JCPenney, Sears and Best Buy.\", ' The mall has over 120 stores as well as a large food court and is 930000 sqft , making it the third largest mall in New Hampshire after the Mall at Rockingham Park in Salem, and the Pheasant Lane Mall in Nashua, which opened in 1991 and 1986, respectively.', ' This was the first large-scale shopping mall in New Hampshire; initial construction of the mall was completed in August 1977, though it has since been dramatically expanded.']], ['Park City Center', ['Park City Center is a shopping mall located in Lancaster, Pennsylvania, and is the largest enclosed shopping center in Lancaster County.', ' It is situated at the intersection of U.S. Route 30 and Harrisburg Pike.', ' The shape of the mall resembles a snowflake, with its stores occupying 8 corridors extending from the center.', ' The roof in the center of the mall is a large white tent, and encloses the octagonal Center Court.', ' The mall underwent a major renovation in 2008, which took 18 months and included updates to every part of the mall.', ' During its early years Park City was also called \"Mall of Four Seasons\" because of the seasonal names given to the 4 corridors leading to each anchor.', \" Going clockwise from west to east was JC Penney in the two-story Winter quadrant, Sears in Spring, Gimbel's (future Pomeroy's/Boscov's) in Summer and Watt & Shand (later Bon-Ton) in Autumn.\", ' The high tech mall located in the heart of Amish country was one of the first to have its own closed-circuit television.', ' Studios for Park City Communications and Lancaster/York/Harrisburg CBS affiliate WLYH-TV 15 were located on the first floor in the Winter wing alongside an ice skating rink.']], ['Dimond Center', ['The Dimond Center is a 728,000 ft² (67,000 m²) shopping mall in Anchorage, Alaska, United States, located on the southwest corner of East Dimond Boulevard and the Old Seward Highway in south Anchorage.', ' It is the largest enclosed mall in the state of Alaska, though the open-air Tikahtnu Commons on the opposite side of town has a greater GLA.']], ['Tower City station', ['Tower City Rapid Station is a rapid transit station in Cleveland, Ohio, USA.', ' It is the central station on the RTA Red Line and the major station on the RTA Green and Blue Lines.', ' The station is located directly beneath Prospect Avenue in the middle of the Tower City Center shopping mall.', ' The station is only accessible through the Tower City Center shopping complex, and, for this reason, the public concourse of the shopping mall is open at all times that the RTA Rapid Transit is in operation.']], ['Pavilion Mall', ['Pavilion Mall is a shopping mall located in Biñan City, Laguna in the Philippines.', ' This is the first large shopping mall in Biñan City, which opened on October, 1999, and is currently being managed by the Ayala Malls, the third largest shopping mall chain in the Philippines.', ' The mall is very accessible via National Highway or via Mamplasan Exit in South Luzon Expressway.']], ['KGOT', ['KGOT (101.3 FM) is a commercial Top 40 (CHR) radio station in Anchorage, Alaska.', ' The station is owned by iHeartMedia, Inc. and broadcasts (along with its sister stations) from studios in the Dimond Center.']], ['SM City Cebu', ['SM City Cebu, also known locally as SM Cebu, is a large shopping mall located in Cebu City, Philippines.', \" It is the 4th shopping mall owned and developed by SM Prime Holdings, the country's largest shopping mall owner and developer.\", \" It is the company's first shopping mall outside of Metro Manila and the 6th largest shopping mall in the Philippines.\", ' It has a land area of 11.8 hectares and a gross floor area of 268,611 m2']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.548\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae685fd5542996d980e7bda', 'answer': 'India Today', 'question': 'Which magazine ranked the Indian Carnatic vocalist who collaborated with classical pianist Anil Srinivasan among 35 Game Changers under 35?', 'supporting_facts': [['Anil Srinivasan', 0], ['Anil Srinivasan', 1], ['Sikkil Gurucharan', 0], ['Sikkil Gurucharan', 4]], 'context': [['G. S. Sareen', ['Gajendra Singh Sareen (born 7 Nov 1966) is a businessman of Indian origin and the founder, CEO and president of Singapore headquartered tire manufacturer and distributor Omni United Pte. Ltd.', ' He founded Omni United in 2003 and has since received numerous entrepreneurship awards including \"Distinguished Business Leader Award\", \"DBS Insignia Spirit of Vision Prestige Award\" and in 2012 was named to Fortune magazine\\'s list of \"Asia\\'s Hottest People in Business\".', ' In 2014, he was featured in \"The Peak\" magazine\\'s anniversary publication as one of 30 men and women documenting their success stories - the \"30/30 – The Game Changers\".']], ['Abhishek Raghuram', ['Abhishek Raghuram (born 1985) is an Indian carnatic vocalist.']], ['M. G. Venkata Raghavan', ['Mysore Govinda Rao Venkata Raghavan (born 14 May 1956) is an Indian Carnatic vocalist, playback singer, composer and actor.', ' He was awarded the Rajyotsava Prashasti, the highhest civilian honour of the state of Karnataka, for his contribution towards Carnatic music.']], ['Dr. Annavarapu Ramaswamy', ['Dr. Annavarapu Ramaswamy (born 23 March 1926) is an Indian Classical Carnatic Violin Vidwan and Guru, whose musical journey entered into 9th decade.', ' He is in the fourth line of direct disciples in Guru Shishya parampara of the Great Musician Saint Thyagaraja and is a disciple of Sri Parupalli Ramakrishnayya Pantulu.', ' Both, Dr.M.Balamuralikrishna and him are very close friends, companions, classmates, learned carnatic music under the same guru Sri Parupalli Ramakrishnayya Pantulu.', ' They performed many concerts together throughout the world.', ' Many stalwarts in carnatic music, wanted and appreciated him as an accompaniment in their concerts in yesteryears.', ' He accompanied top ranked legendary stalwart musicians of India, carnatic musicians such as his guru, Sri Parupalli Ramakrishnayya Pantulu, \\xa0Dr. Mangalampalli Balamurali Krishna, Sri Arayakudi Ramanuja Iyyengar, Sri Chembai Vaidyanatha Bhagavatar, Sri G. N. Balasubramaniam, Sri Semmamgudi Srinivasa Iyyer, T. R. Maralingam, Sri S.Balachandar etc..', '.', '; and Hindustani Musicians - Pandit Vinayakarao Pathvardhan, Pandit Bhimsen Joshi, Pandit Jasraj, etc., on many Prestigious National and International venues.', ' He came up with his own style of performing solo concerts on Violin & Viola.', ' He invented new Ragas & Talams such as Vandana Ragam, SriDurga Ragam and Tinetradi Tala and Vedadi Tala.', ' He had written and composed many Varnams and Kritis.', ' He is a notable guru, taught many students, who are in successful positions such as Violin Vasu, Flute Phani, V.L.Tulasi Viswanath (carnatic vocalist), Peravali Nanda Kumar (Violinist), etc., For the past 7 to 8 decades, he has been doing free service to the society such as teaching students at free of cost.', ' He propagated the magnificence of his performances, teachings, demonstrations and Lectures on Music in countries such as U. S. A, Canada, European Countries (U. K, France etc.) and Asian Countries (Muscat, Bahrain, Dubai, Singapore, Malaysia, Doha, Srilanka, etc.,)']], ['Sikkil Gurucharan', ['Sikkil C. Gurucharan (born on 21 June 1982) is among the foremost young performing musicians of Carnatic music in India today.', ' He is the grandson of Sikkil Kunjumani, elder of the internationally acclaimed flautists the Sikkil Sisters.', ' Gurucharan has been under the tutelage Vaigal Shri S. Gnanaskandan and is currently being mentored by Shri B. Krishnamurthy.', \" He is an 'A' grade All India Radio artist.\", ' The magazine India Today featured him among 35 Game Changers Under (the age of) 35 in India, a list of young achievers from different walks of life.']], ['Gayathri Venkataraghavan', ['Gayathri Venkataraghavan (Tamil: காயத்ரி வெங்கட்ராகவன்) is an Indian Carnatic vocalist.', ' She lives in Chennai.']], ['Nedunuri Krishnamurthy', ['Nedunuri Krishnamurthy (10 October 1927 – 8 December 2014) was an Indian Carnatic vocalist.', \" He was awarded the Madras Music Academy's Sangeetha Kalanidhi in 1991.\"]], ['Anil Srinivasan', ['Anil Srinivasan (born June 3, 1977) is a classical pianist from India.', ' Born in Chennai, India and educated at the University of Southern California and at Columbia University, New York, he is well known for his collaborative work with Carnatic vocalist Sikkil Gurucharan.', ' Equally known is his pioneering work in music education in South India.']], ['Neyveli Santhanagopalan', ['Neyveli Santhanagopalan (born 1963) is an Indian Carnatic vocalist.']], ['Charanams', ['Charanams is a world music carnatic jazz band that presents a unique blend of South Indian carnatic music and jazz music.', ' Charanams band presents musical compositions of carnatic musician Nivedita ShivRaj.', ' These music compositions are based on South Indian Carnatic music with jazz improvisations.', ' The compositions do not have any lyrics, but contains carnatic music solfege.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.549\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a81a4615542995ce29dcc1e', 'answer': 'sub-Saharan Africa', 'question': 'Where is the closest relative to the whale located?', 'supporting_facts': [['Whale', 1], ['Hippopotamus', 0]], 'context': [['Crossosperma', ['Crossosperma is a genus of shrubs in the family Rutaceae.', ' The genus is endemic to New Caledonia in the Pacific and contains two species.', ' Its closest relative is the Australian \"Acradenia\".']], ['Hachettea', ['Hachettea austrocaledonica is a species of parasitic plant in the Balanophoraceae family.', ' It is endemic to New Caledonia and the only species of the genus Hachettea.', ' Its closest relative is \"Dactylanthus\" from New Zealand.']], ['Bombyx mandarina', ['Bombyx mandarina, the wild silkmoth, is an insect from the moth family Bombycidae.', ' It is the closest relative of \"Bombyx mori\" the domesticated silkmoth or \"silkworm\" (properly, this refers to the caterpillars only).', ' Unlike the domesticated relative which is unable to fly or indeed persist outside human care, the wild silkmoth is a fairly ordinary lepidopteran. Its main difference from the domesticated taxon is the more slender body with well-developed wings in males, and the dull greyish-brown color.']], ['Hippopotamus', ['The common hippopotamus (\"Hippopotamus amphibius\"), or hippo, is a large, mostly herbivorous and aggressive mammal in sub-Saharan Africa, and one of only two extant species in the family Hippopotamidae, the other being the pygmy hippopotamus (\"Choeropsis liberiensis\" or \"Hexaprotodon liberiensis\").', ' The name comes from the ancient Greek for \"river horse\" (ἱπποπόταμος ).', ' After the elephant and rhinoceros, the common hippopotamus is the third-largest type of land mammal and the heaviest extant artiodactyl.', ' Despite their physical resemblance to pigs and other terrestrial even-toed ungulates, the closest living relatives of the Hippopotamidae are cetaceans (whales, dolphins, porpoises, etc.) from which they diverged about .', ' The common ancestor of whales and hippos split from other even-toed ungulates around .', ' The earliest known hippopotamus fossils, belonging to the genus \"Kenyapotamus\" in Africa, date to around .']], ['Daenikera', ['Daenikera corallina is a species of parasitic in the Santalaceae family.', ' It is endemic to New Caledonia and the only species of the genus Daenikera.', ' Its closest relative is \"Amphorogyne\", also endemic to New Caledonia.']], ['Occidens (tetrapod)', ['Occidens is an extinct genus of stem tetrapod that lived during the earliest part of the Carboniferous in what is now Northern Ireland.', ' It is known from a single type species, Occidens portlocki, named in 2004 on the basis of a left lower jaw that British geologist Joseph Ellison Portlock described in 1843.', ' Portlock attributed it to the lobe-finned fish \"Holoptychius\" and it was housed in the collections of the British Geological Survey for over a century before being reevaluated in 2004 by vertebrate paleontologists Jenny Clack and Per E. Ahlberg, who reclassified it as a new genus and species of early tetrapod.', ' The genus name \"Occidens\" refers to its presence west of better-known early tetrapod assemblages in Great Britain, and the species name honors Portlock.', ' The jaw likely comes from the Altagoan Formation and, based on an analysis of fossilized pollen, dates to the late Tournaisian stage of the Early Carboniferous about 350 million years ago.', ' The occurrence of \"Occidens\" in the Tournaisian makes it a critically important taxon because it lies within Romer\\'s gap, a time interval spanning most of the Early Carboniferous in which almost no tetrapod fossils are known.', \" Romer's gap separates the first appearance of tetrapods in the Late Devonian from the group's first evolutionary radiation toward the end of the Early Carboniferous.\", ' However, the relationship of \"Occidens\" to other early tetrapods both before and after the gap remain uncertain, which means that its context in tetrapod evolution remains unknown.', ' Clack and Ahlberg noted several distinctive features of \"Occidens\", including a straight row of teeth along the coronoid bones on the inner surface of the lower jaw, an open groove for a lateral line sense organ on the jaw\\'s outer surface, and a stepped shape to the connection between the dentary and angular bones.', ' The jaw bone is deep, resembling those of \"Crassigyrinus\" and whatcheeriids (which both occur in Romer\\'s gap) in overall appearance.', ' In most phylogenetic trees produced by Clack and Ahlberg\\'s 2004 analysis, \"Occidens\" fell near whatcheeriids and the Devonian taxon \"Tulerpeton\", being more derived than all other Devonian taxa and more basal than \"Crassigyrinus\" and the post-Romer\\'s Gap taxa \"Greererpeton\" and \"Megalocephalus\".', ' A 2008 phylogenetic analysis by paleontologists Marcello Ruta and John Bolt found \"Occidens\" to be the closest relative of \"Sigournea multidentata\", a species from the end of the gap found in Iowa, but could not determine where these two taxa fit relative to other Early Carboniferous tetrapods.']], ['Carusia', ['Carusia is an extinct genus of lizard from the Late Cretaceous of Mongolia.', ' It is a close relative of the family Xenosauridae, which includes living knob-scaled lizards. Fossils of the type and only species Carusia intermedia come from the late-Campanian age Barun Goyot Formation and have been found in the Flaming Cliffs, Ukhaa Tolgod, and Kheerman Tsav fossil localities.', ' \"Carusia\" was first described in 1985 under the name \"Carolina intermedia\", but since the name \"Carolina\" was preoccupied by a genus of scarab beetles that had been named in 1880, it was renamed \"Carusia intermedia\".', ' \"Carusia\" had initially been known from fragmentary skull material, complicating efforts to determine its evolutionary relationships with other lizards; it had variously been described as an indeterminate scincomorph, a xenosaurid, or some other type of autarchoglossan lizard convergent with xenosaurids.', ' However, the discovery of 35 complete skulls in the 1990s, three of which were described in a detailed 1998 monograph, revealed that \"Carusia\" was the sister taxon (closest relative) of Xenosauridae, compelling the authors of the monograph to create a new clade called Carusioidea to include both taxa.']], ['Mascarene teal', ['The Mascarene teal (\"Anas theodori\"), also known as Sauzier\\'s teal and Mauritian duck, is an extinct dabbling duck that formerly occurred on the islands of Mauritius and Réunion.', ' It was a small teal of the \"Anas gibberifrons\" superspecies of the \"Anas\" subgenus \"Nettion\".', \" Its closest relative is probably Bernier's teal from Madagascar and, apart from having stronger wings and being considerably bigger (between a Sunda teal and a mallard in size), it seems to have looked very similar to that species.\", ' Earlier, it was proposed that Meller\\'s duck, also from Madagascar, is the closest living relative of \"A. theodori\", but as more remains of the latter were unearthed this appears far less likely.', ' Apart from a few, brief descriptions, not much is known about the bird in life, but its habits probably did not differ significantly from those of its close relatives.', ' Bones have been found in the Mare aux Songes swamp on Mauritius and more recently on Réunion also.', ' The scientific name honours Thé́odore Sauzier, who made many bones of extinct birds found on Mauritius available to science.']], ['Gerobatrachus', ['Gerobatrachus is an extinct genus of amphibamid temnospondyl (represented by the type species Gerobatrachus hottoni) that lived in the Early Permian, approximately 290 million years ago (Ma), in the area that is now Baylor County, Texas.', ' When it was first described in 2008, \"Gerobatrachus\" was announced to be the closest relative of Batrachia, the group that includes modern frogs and salamanders.', ' It possesses a mixture of characteristics from both groups, including a large frog-like head and a salamander-like tail.', ' These features have led to it being dubbed a frogamander by the press.', ' Some more recent studies place \"Gerobatrachus\" as the closest relative of Lissamphibia, the group that contains all modern amphibians including frogs, salamanders, and caecilians, or place modern amphibians far from \"Gerobatrachus\" within a group called Lepospondyli.']], ['Whale', ['Whales are a widely distributed and diverse group of fully aquatic placental marine mammals.', ' They are an informal grouping within the infraorder Cetacea, usually excluding dolphins and porpoises. Whales, dolphins and porpoises belong to the order Cetartiodactyla with even-toed ungulates and their closest living relatives are the hippopotamuses, having diverged about 40 million years ago.', ' The two parvorders of whales, baleen whales (Mysticeti) and toothed whales (Odontoceti), are thought to have split apart around 34 million years ago.', ' The whales comprise eight extant families: Balaenopteridae (the rorquals), Balaenidae (right whales), Cetotheriidae (the pygmy right whale), Eschrichtiidae (the grey whale), Monodontidae (belugas and narwhals), Physeteridae (the sperm whale), Kogiidae (the dwarf and pygmy sperm whale), and Ziphiidae (the beaked whales).']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.550\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab1dcab5542993be8fa985e', 'answer': 'Marie', 'question': 'What was the middle name of the actress who was cast alongside Harvey Fierstein, Linda Hart, Dick Latessa, Clarke Thorell, Mary Bond Davis, Laura Bell Bundy, Matthew Morrison, Corey Reynolds, and Marissa Jaret Winokur in Hairspray?', 'supporting_facts': [['Hairspray (2002 album)', 2], ['Kerry Butler', 0]], 'context': [['Beautiful Girl (film)', ['Beautiful Girl is a 2003 television movie starring Marissa Jaret Winokur.', ' The film was directed by Douglas Barr for the ABC Family network.']], ['Dance Your Ass Off (season 1)', ['The first season of Dance Your Ass Off aired from June 29, 2009 to September 7, 2009.', ' It aired on the Oxygen Network.', ' It was the only season to feature Marissa Jaret Winokur as host.', ' The show featured twelve overweight contestants competing to dance and lose weight.', ' The medical doctor was Rob Huizenga from the USA \"Biggest Loser\".', ' For this season, the judges were Danny Teeson, Lisa Ann Walter and Mayte Garcia.']], ['Retired at 35', ['Retired at 35 is an American sitcom on TV Land starring George Segal, Jessica Walter, Johnathan McClain, Josh McDermitt, Marissa Jaret Winokur, and Ryan Michelle Bathe.', ' It is the network\\'s second original scripted series after \"Hot in Cleveland\".', ' The series premiered on January 19, 2011.', ' On March 21, 2011, the series was renewed for a second season.', ' The second season premiered on Tuesday, June 26, 2012, at 10:00 pm ET/PT, and concluded on Wednesday, August 29, 2012.']], ['Giddy On Up', ['\"Giddy On Up\" is the debut single by American stage actress and singer Laura Bell Bundy.', ' Co-written by Bundy, it was released to country music radio in February 2010 as the lead-off single from her debut album \"Achin\\' and Shakin\\',\" which was released on April 13, 2010.', ' Bundy wrote this song with Jeff Cohen and Mike Shimshack.']], ['Dance Your Ass Off', ['Dance Your Ass Off (also rendered in a censored form as Dance Your A** Off for broadcast television mentions and promotions) is a reality competition series on the Oxygen Network hosted by Marissa Jaret Winokur in the first season, then Melanie Brown in season two.', ' Similar to the set up of \"Dancing with the Stars\" competitors are paired with a professional dancers in hope of impressing judges and the viewing audience.', ' However, each of the twelve contestants are also hoping to lose weight during the process.', ' It premiered on June 29, 2009.', ' The medical doctor is Rob Huizenga from \"The Biggest Loser\".', ' The season premiere brought in 4.3 million viewers making it the most watched show in history of Oxygen Network.', ' In the judges panel are Danny Teeson, a lifestyle coach and dancing expert, actress Lisa Ann Walter, and professional dancer Mayte Garcia, who only appeared as a guest judge for a week in season 2.']], ['Surf School', ['Surf School is a 2006 American teen sex comedy written and directed by Joel Silverman.', ' It stars Corey Sevier, Laura Bell Bundy, Harland Williams, and Sisqó.', ' A group of misfits must learn to surf in one week so they can compete in the championships.']], [\"Achin' and Shakin'\", ['Achin\\' and Shakin\\' is the second studio album released by \"Broadway\" actress and Mercury Nashville recording artist Laura Bell Bundy.', ' The album, which was released on April 13, 2010, is Bundy\\'s first mainstream album; her first album, \"Longing for a Place Already Gone\", was self-released in 2007.', ' \"Achin\\' and Shakin\"\\' features the singles \"Giddy On Up\" and \"Drop On By\".']], ['Kerry Butler', ['Kerry Marie Butler (born June 18, 1971) is an American actress known primarily for her work in theatre.']], ['Marissa Jaret Winokur', ['Marissa Jaret Winokur (born February 2, 1973), sometimes credited as Marissa Winokur, is an American actress known for her Tony-winning performance as Tracy Turnblad in the highly successful Broadway musical adaptation of John Waters\\' film \"Hairspray,\" as well as her work on the Pamela Anderson sitcom \"Stacked.\"', ' Some of her other TV credits include \"Curb Your Enthusiasm,\" \"Moesha,\" \"The Steve Harvey Show,\" \"Just Shoot Me!', ',\" \"Felicity,\" and \"Dharma & Greg.\"']], ['Hairspray (2002 album)', ['Hairspray: Original Broadway Cast Recording is the cast album for the 2002 musical \"Hairspray\".', ' The show is an adaptation of the 1988 film of the same name.', \" It features performances from the show's cast, which includes Harvey Fierstein, Linda Hart, Dick Latessa, Kerry Butler, Clarke Thorell, Mary Bond Davis, Laura Bell Bundy, Matthew Morrison, Corey Reynolds, and Marissa Jaret Winokur as the lead character of Tracy Turnblad.\", ' The cast recording earned the 2003 Grammy Award for Best Musical Theater Album.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.551\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adca5eb5542994d58a2f693', 'answer': \"Bishop's Stortford\", 'question': 'Kevin Watson currently manages which Southern League football team?', 'supporting_facts': [['Kevin Watson', 1], [\"Bishop's Stortford F.C.\", 1]], 'context': [['List of Plymouth Argyle F.C. seasons', ['Plymouth Argyle Football Club is an English association football club based in Plymouth, Devon.', ' They compete in Football League Two, the fourth tier of the English football league system, as of the 2012–13 season.', ' The club was founded in 1886 as \"Argyle Football Club\".', ' At this time, there was no League football, so matches were arranged on an ad hoc basis, supplemented by cup competitions.', ' In January 1903, the club became a limited company and changed their name to \"Plymouth Argyle F.C.\" Election to the Southern League followed in March, as the club gained direct entry to the First Division for the 1903–04 season.', ' The club were also invited to compete in the Western League, a competition which was considered secondary to the Southern League.']], ['1946–47 Colchester United F.C. season', [\"The 1946–47 season was Colchester United's fifth season in their history and their fifth in the Southern League.\", ' Alongside competing in the Southern League, the club also participated in the FA Cup and Southern League Cup.', ' New manager Ted Fenton began to assemble a team of professionals following the reliance on guest players during the 1945–46 season, as the club finished 8th in the league.', ' They reached the first round of the FA Cup, but were defeated by Football League side Reading.', ' They were also Southern League Cup semi-finalists, defeated at Priestfield Stadium by Gillingham.']], [\"Bishop's Stortford F.C.\", [\"Bishop's Stortford Football Club is a football club based in Bishop's Stortford, Hertfordshire, England.\", ' They are currently members of the Southern League Premier Division and play at Woodside Park.']], ['Dover Athletic F.C.', ['Dover Athletic Football Club is an association football club based in the town of Dover, Kent, England.', ' The club currently competes in the National League, the fifth tier of English football.', \" The club was formed in 1983 after the dissolution of the town's previous club, Dover F.C., whose place in the Southern League was taken by the new club.\", \" In the 1989–90 season Dover Athletic won the Southern League championship, but failed to gain promotion to the Football Conference as the club's ground did not meet the required standard.\", ' Three seasons later the team won the title again and this time gained promotion to the Conference, where they spent nine seasons before being relegated at the end of the 2001–02 season.', \" The club was transferred from the Southern League to the Isthmian League in 2004, competing in that league's Premier Division for one season before mounting financial problems led the club to a further relegation.\", ' In the 2007–08 season, Dover won Division One South of the league, before winning the Premier Division in 2008–09 and thus gaining promotion to the Conference South.', ' They spent five seasons in this division, reaching the play-offs three times, before defeating Ebbsfleet United in the 2013–14 play-off final to finally return to the Conference Premier after a twelve-year absence.']], ['1914–15 Watford F.C. season', ['Watford Football Club are an association football team from the county of Hertfordshire, England.', ' The 1914–15 season was their nineteenth season of league football, since joining the Southern League as West Hertfordshire for the 1896–97 season.', ' Watford finished the season as champions of the Southern League First Division, winning 22 and drawing 8 of their 38 league matches.', ' In other competitions, Watford were eliminated from the FA Cup in the sixth qualifying round by Rochdale, and from the Southern Charity Cup by fellow Southern League team Luton Town.', \" The club's manager was Harry Kent, and its top scorer George Edmonds, with 17 goals from 35 appearances.\", \" Other notable players included Skilly Williams, who began what would be a 13-year period as the club's first choice goalkeeper, and Fred Gregory, whose goal against Gillingham sealed the title for Watford.\", \" Gregory and Williams were also the only two men to play in all 40 of Watford's games.\"]], ['List of Millwall F.C. seasons', ['This is a list of all seasons played by Millwall Football Club from their early beginnings in the Southern League, to their inaugural season in 1920–21 in the English Football League and up to their last completed season.', ' It details their record in the FA Cup, the League Cup and other major competitions entered, as well as managers, top goalscorers and average home attendance for each season.', ' Millwall were founded in 1885 and for the first nine years of their existence did not compete in league football.', ' They first entered the FA Cup in 1887, turning full-time professional as a club in 1890.', ' They were founding members of the Southern League in 1894, which they competed in for 22 seasons, claiming the title twice.', ' They left to join the Football League in 1920.', ' Millwall have played in all four divisions during their 91 seasons in the league, including the Third Division South, which they won twice, in 1928 and 1938.', ' Millwall were Fourth Division champions in 1962.']], ['Kevin Watson', ['Kevin Edward Watson (born 3 January 1974) is a former professional footballer, turned coach and sports broadcaster.', \" He is currently the manager of Bishop's Stortford.\"]], ['Poole Town F.C.', ['Poole Town Football Club is a football club based in Poole, Dorset, England.', ' They currently compete in the National League South, the sixth tier of English football, after being promoted as champions of the Southern League Premier Division in the 2015–16 season.', ' They were established in 1880 and joined the Western League Division Two in 1930.', ' The club is affiliated to the Dorset County Football Association and is a FA Charter Standard Community Club.', ' They won the Western League title in 1957 and reached the First Round Proper of the FA Cup four times in their history and the Third Round once, losing to Everton at Goodison.', ' They play at the Black Gold Stadium, at Tatnam, Poole, and finished the 2008–09 season as \"Double\" winners – champions and Dorset Senior Cup winners.', ' In 2009–10 they became back-to-back champions of the Wessex League and completed an unprecedented treble winning the League for the 3rd time in a row in 2010–11.', ' Promotion to the Southern League was finally achieved after an upgrade to the Tatnam facilities.', ' Poole were second in their first season, losing to Gosport in a playoff final and then Champions and promoted to the Southern League Premier Division the following season.', ' In 2015-2016 they were promoted to the National League South after finishing as champions of the Southern League.', ' In the 2016/17 season there were on course to finish in the playoffs after a prolonged stay in the top 7 places.', ' After it was announced they could not participate in the playoff games due to not meeting ground grading requirements by 21st March they suffered a dip in form.', ' A late resurgence ensured they finished 5th and had to relinquish the playoff place they had subsequently earned.']], ['1919–20 Watford F.C. season', ['Watford Football Club is a association football team from the county of Hertfordshire, England.', ' The 1919–20 season was their twentieth season of league football, and their first since 1914–15 due to the outbreak of the First World War.', ' It was also their final season in the Southern League, having originally joined it as West Hertfordshire for the 1896–97 season, prior to a merging with another club and renaming in 1898.', ' Having started the season as reigning champions, Watford finished the season as runners up of the Southern League First Division on goal average.', \" They won 26 and drew 6 of their 42 league matches, compared to eventual champions Portsmouth's record of 23 wins and 12 draws.\", \" Watford's only other competitive fixture was in the FA Cup, where they were eliminated in the 6th Qualifying round by fellow Southern League side Southend United.\", \" The club's manager was Harry Kent, and its top scorer was George Edmonds, with 19 goals from 37 appearances.\"]], [\"1896–97 Southampton St. Mary's F.C. season\", [\"The 1896–97 season was the twelfth since the foundation of Southampton St. Mary's F.C. and their third in league football, as members of the Southern League.\", \" The season was the most successful yet, with St. Mary's claiming the Southern League title for the first time and reaching the Second Round Proper of the FA Cup.\", \" It was the start of the most successful period in the club's history — in a period of eight years, they were Southern League champions six times and reached the final of the FA Cup twice.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.551\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adddd075542992200553b71', 'answer': 'Wes Archer', 'question': 'Season 2 of Rick and Morty began with the episode directed by whom?', 'supporting_facts': [['Rick and Morty (season 2)', 0], ['A Rickle in Time', 1]], 'context': [['Lawnmower Dog', ['\"Lawnmower Dog\" is the second episode of \"Rick and Morty\".', ' It premiered on Adult Swim on December 9, 2013, was written by Ryan Ridley, and directed by John Rice.', \" In the episode, Rick gives Jerry a device to enhance the dog's intelligence, whilst Rick and Morty get lost in the dreams of Morty's math teacher.\", ' The episode was well received, with approximately 1.5 million viewers when airing.']], ['Rixty Minutes', ['\"Rixty Minutes\" is the eighth episode of the first season of \"Rick and Morty\".', ' It premiered on Adult Swim on March 17, 2014.', ' The episode was written by Tom Kauffman and Justin Roiland, and directed by Bryan Newton.', ' In the episode, Rick and Morty watch cable from other dimensions, while Jerry, Beth, and Summer watch alternate reality versions of themselves using a pair of interdimensional goggles.', ' The episode was well received and watched by about 1.48 million viewers in the United States.']], ['The Ricks Must Be Crazy', ['\"The Ricks Must Be Crazy\" is the sixth episode in the second season of the American animated television sitcom \"Rick and Morty\", and the seventeenth overall episode in the series.', ' Written by Dan Guterman and directed by Dominic Polcino, the episode first aired on Adult Swim in the United States on August 30, 2015.', ' It is speculated that the title of the episode is a reference to the 1980 film \"The Gods Must Be Crazy\".']], ['Rick Potion No. 9', ['\"Rick Potion #9\" is the sixth episode of \"Rick and Morty\".', ' It premiered on Adult Swim on January 27, 2014, was written by Justin Roiland, and directed by Stephen Sandoval.', ' In the episode, a love potion goes wrong, creating a virus that begins to infect the entire world population, making everyone fall in love with Morty.', ' The episode has been well received, and was seen by about 1.7 million viewers when airing.', ' The title of the episode is in reference to the 1959 song \"Love Potion No. 9\" by the Clovers.']], ['Pilot (Rick and Morty)', ['\"Pilot\" is the first episode of \"Rick and Morty\".', ' It premiered on Adult Swim on December 2, 2013.', ' It is written by series creators Dan Harmon and Justin Roiland, and directed by Roiland.', ' The series introduces protagonists, alcoholic scientist Rick Sanchez and his innocent teenage grandson Morty Smith, as they embark on a dangerous interdimensional adventure to fetch Mega tree seeds.', ' The pilot had a mixed to positive reception and was seen by about 1.1 million viewers when airing.']], ['Gangland Undercover', ['Gangland Undercover is an American factual based drama television series written and created by Executive Producer Stephen Kemp and co-writer Noel Baker.', ' It was inspired by the story of Charles Falco a former ATF confidential informant (CI) who infiltrated an outlaw motorcycle club.', ' The series is based on Falco\\'s 2013 memoir, \"Vagos, Mongols, and Outlaws: My Infiltration of America\\'s Deadliest Biker Gangs\".', ' It premiered on Tuesday, February 24, 2015, on the History Channel at 10/9c.', ' In Canada, Season 2 premiered on September 26, 2016.', ' In the U.S., only the first episode of season 2 aired on December 8, 2016 on A&E.', ' According to the trailer, \"the new season\" (Season 2) premiered on March 2, 2017.']], ['A Rickle in Time', ['\"A Rickle in Time\" is the first episode in the second season of the American animated television sitcom \"Rick and Morty\", and the twelfth overall episode of the series.', ' Written by Matt Roller and directed by Wes Archer, the episode first aired on Adult Swim in the United States on July 26, 2015.']], ['Angry Birds Stella (TV series)', ['Angry Birds Stella is a Finnish computer-animated TV series based on the game \"Angry Birds Stella\" that was produced by Rovio Entertainment.', ' The first episode, \"A Fork in the Friendship\", aired on ToonsTV in November 1, 2014.', ' The series recounts the tale of Stella, along with her friends Luca, Willow, Poppy and Dahlia, as they work their way against Gale, the former friend of Stella, that is the queen of the pigs in Golden Island.', ' The first season focuses on Gale trying to hunt for the Golden Egg, but, fails as seen in the final episode of season 1, \"To The Bitter End\".', ' The second season focuses on Gale returning and trying to hunt for the Golden Egg again, and she succeeded as seen in \"The Golden Queen\", the 9th episode of season 2.', ' Also, Dahlia, one of the birds, tries searching for the egg as well for an experiment, but this time, she fails, as seen in \"It\\'s Mine!\"', ', the 6th episode of season 2.', ' Soon enough, the birds realize how dangerous this egg can be, as seen in \"Premonition\", the 11th episode in season 2, which in when Gale with the Golden Egg, anything that is touched with it, turns to gold.', ' However, when that happens, another thing turns to stone, the opposite of gold, which causes nature, and even worse, food to be turned to stone as well.', ' Now, it is up to the Stella gang to save Golden Island before it is too late.', ' They do succeed, as seen in \"You Asked For It\", the 13th and final episode of season 2 and the series finale of \"Angry Birds Stella\", where after a long awful night, Stella and her gang launch the Golden Egg out of the island, never to be seen again.', ' Gale does not mind because when she saw the Golden Egg after she broke out of the gold in the episode, she ran away, back into her castle.']], ['Rick and Morty (season 2)', ['The second season of the animated television series \"Rick and Morty\" originally aired in the United States on Cartoon Network\\'s late night programming block, Adult Swim, which premiered on July 26, 2015 with \"A Rickle in Time\", and concluded on October 4 with \"The Wedding Squanchers\".', ' This season aired a total of ten episodes.']], ['Meeseeks and Destroy', ['\"Meeseeks and Destroy\" is the fifth episode of the first season of \"Rick and Morty\".', ' It premiered on Adult Swim on January 20, 2014.', ' The episode was written by Ryan Ridley and directed by Bryan Newton.', ' In the episode, Rick provides the family with a solution to their problems, freeing him up to go on an adventure led by Morty.', '\\xa0The episode has been well received, and was seen by about 1.6 million viewers when it was first aired on the Adult Swim channel.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.553\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a75d00a5542992db94736d1', 'answer': 'Rhodesia', 'question': 'Which state in Southern Africa, whose capital is located in Salisbury, hosted the Currie Cup in which John David McPhun played?', 'supporting_facts': [['John McPhun', 0], ['Rhodesia', 1]], 'context': [['1996 Currie Cup', [\"The 1996 Currie Cup was the 58th season of the Currie Cup, South Africa's premier domestic rugby union competition, since it started in 1889.\", ' The competition was known as the Bankfin Currie Cup for sponsorship reasons and was contested from 30 May to 24 October 1996.', ' This was also the first season since the advent of professionalism in South African rugby union, which led to a major restructuring in several facets of the sport.', ' The number of provincial unions were reduced from 22 to 14, all of which participated in a single Currie Cup tournament.']], ['1991 Currie Cup', ['The 1991 Currie Cup (known as the \"Bankfin Currie Cup\" for sponsorship reasons) was the top division of the Currie Cup competition, the premier domestic rugby union competition in South Africa.', \" This was the 53rd season since the competition started in 1889 and the first time it was known as the Bankfin Currie Cup, following the sponsors' name change from Santam Bank.\"]], ['Rhodesia', ['Rhodesia ( ), commonly known from 1970 onwards as the Republic of Rhodesia, was an unrecognised state in southern Africa from 1965 to 1979, equivalent in territorial terms to modern Zimbabwe.', ' With its capital in Salisbury (now Harare), Rhodesia was considered a \"de facto\" successor state to the former British colony of Southern Rhodesia (which had achieved responsible government in 1923).']], ['2003 Currie Cup', ['The 2003 Currie Cup was the 2003 season of the South African domestic rugby union competition, the Absa Currie Cup premier divisison, played from 26 July 2003 - 1 November 2003.', ' The 2003 Currie Cup saw the implementation of a new format for the tournament with the Cup being split into 2 divisions, the Premier Division and a lower division.', ' The Premier Division consisting of the 6 top provincial teams and the lower division consisting of 8 teams for a total of 14 teams participating in the Currie Cup.', ' The teams in the divisions played matches among themselves with top teams progressing to the finals.', ' The finals were played at Securicor Loftus Stadium where the Blue Bulls beat the Sharks 40-19 to win the Cup.', ' This would be the second win for the Blue Bulls in a streak of 3 consecutive Currie Cup wins from 2002-2004.', \" The Blue Bull's Ettienne Botha scored two tries in the final.\", ' This equaled the record for tries scored in a Currie Cup final at the time.']], ['1992 Currie Cup / Central Series', ['The 1992 Currie Cup / Central Series was a rugby union competition held between the teams in the 1992 Currie Cup and 1992 Currie Cup Central A competitions, the top two tiers of the premier domestic competition in South Africa.', ' This formed part of the 54th Currie Cup season since the competition started in 1889.']], ['1991 Currie Cup / Central Series', ['The 1991 Currie Cup / Central Series was a rugby union competition held between the teams in 1991 Currie Cup and 1991 Currie Cup Central A competitions, the top two tiers of the premier domestic competition in South Africa.', ' This formed part of the 53rd Currie Cup season since the competition started in 1889.']], ['2016 Currie Cup qualification', ['The 2016 Currie Cup qualification series was a South African rugby union competition organised by the South African Rugby Union which was played between 9 April and 23 July 2016.', \" It featured all fourteen South African provincial unions plus the Welwitschias from Namibia and served as a qualifying competition for the 2016 Currie Cup, the 78th edition of South Africa's premier domestic rugby union competition.\", ' Nine teams from this competition advanced to the 2016 Currie Cup Premier Division, while the remaining six teams progressed to the 2016 Currie Cup First Division.']], ['Currie Cup / Central Series', ['The Currie Cup / Central Series was a rugby union competition held between the Currie Cup and Currie Cup Central A teams, the top two tiers of the premier domestic competition in South Africa.', ' This formed part of the 1986-1994 Currie Cup seasons.']], ['John McPhun', ['John David McPhun (born September 8, 1940 in Salisbury, Rhodesia) was a first-class cricketer who played for Rhodesia in the Currie Cup.']], ['1992 Currie Cup Central / Rural Series', ['The 1992 Currie Cup Central / Rural Series was a rugby union competition held between the teams in the 1992 Currie Cup Central B and 1992 Currie Cup Rural A competitions, the third and fourth tiers of the premier domestic competition in South Africa.', ' This formed part of the 54th Currie Cup season since the competition started in 1889.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.553\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8a764555429930ff3c0de1', 'answer': 'American-born', 'question': \"What distinction about his birth makes the contemporary of Bill Pollack unique as a winner of the Formula One World Drivers' Championship?\", 'supporting_facts': [['Bill Pollack', 0], ['Bill Pollack', 3], ['Phil Hill', 0]], 'context': [['2015 FIA Formula One World Championship', ['The 2015 FIA Formula One World Championship was the 69th season of FIA Formula One motor racing.', \" It featured the 2015 Formula One World Championship, a motor racing championship for Formula One cars, recognised by the sport's governing body, the Fédération Internationale de l'Automobile (FIA), as the highest class of competition for open-wheel racing cars.\", \" Twenty-two drivers representing ten teams contested nineteen Grands Prix, starting in Australia on 15 March and ending in Abu Dhabi on 29 November as they competed for the World Drivers' and World Constructors' championships.\"]], ['Phil Hill', [\"Philip Toll Hill Jr. (April 20, 1927 – August 28, 2008) was an American automobile racer and the only American-born driver to win the Formula One World Drivers' Championship (Mario Andretti, an Italian American driver, won the World Drivers' Championship in 1978, but was not born in the United States).\", ' He also scored three wins at each of the 24 Hours of Le Mans and 12 Hours of Sebring sports car races.']], ['2016 FIA Formula One World Championship', [\"The 2016 FIA Formula One World Championship was the 70th season of the Fédération Internationale de l'Automobile (FIA)'s Formula One motor racing.\", \" It featured the 67th Formula One World Championship, a motor racing championship for Formula One cars which is recognised by the sport's governing body, the FIA, as the highest class of competition for open-wheel racing cars.\", \" Teams and drivers took part in twenty-one Grands Prix—making for the longest season in the sport's history—starting in Australia on 20 March and finishing in Abu Dhabi on 27 November as they competed for the World Drivers' and World Constructors' championships.\"]], ['2019 FIA Formula One World Championship', [\"The 2019 FIA Formula One World Championship is a planned motor racing championship for Formula One cars which would be recognised by the governing body of international motorsport, the Fédération Internationale de l'Automobile (FIA), as the highest class of competition for open-wheel racing cars.\", ' It would be the 70th Formula One World Championship and the 73rd season of Formula One motor racing.', \" Teams and drivers would be competing for the World Constructors' and World Drivers' championships.\", ' 2019 is also scheduled to see the one thousandth World Championship Grand Prix.']], ['Bill Pollack', ['Bill Pollack was a competitor in the early days of the post World War II California sports car culture.', \" Starting his racing career in 1950, he would go on to win the Pebble Beach in an Allard J2 that was owned by Tom Carstens'.\", ' Bill also won at other California venues of that time: Pebble Beach, Golden Gate Park, Reno, Torrey Pines, Stockton, Madera, Willow Springs, Palm Springs, and the Santa Barbara road races.', ' He was a contemporary of Phil Hill, who went on to become the first and only U.S. born world grand prix champion.']], ['1987 FIA Formula One World Championship', ['The 1987 FIA Formula One World Championship was the 41st season of FIA Formula One motor racing.', ' It featured the 1987 Formula One World Championship for Drivers and the 1987 Formula One World Championship for Constructors, both of which commenced on 12 April 1987 and ended on 15 November after sixteen races.', ' The World Championship for Drivers was won by Nelson Piquet, and the World Championship for Constructors by Williams-Honda.', ' The season also encompassed the Jim Clark Trophy and the Colin Chapman Trophy, which were respectively contested by drivers and constructors of Formula One cars powered by naturally aspirated engines.']], ['2017 FIA Formula One World Championship', ['The 2017 FIA Formula One World Championship is the 71st season of Formula One motor racing.', \" It features the 68th Formula One World Championship, a motor racing championship for Formula One cars which is recognised by the sport's governing body, the Fédération Internationale de l'Automobile (FIA), as the highest class of competition for open-wheel racing cars.\", \" Teams and drivers are competing in twenty Grands Prix—starting in Australia on 26 March and ending in Abu Dhabi on 26 November—for the World Drivers' and World Constructors' championships.\"]], ['2014 FIA Formula One World Championship', ['The 2014 FIA Formula One World Championship was the 68th season of FIA Formula One motor racing.', \" It featured the 2014 Formula One World Championship, a motor racing championship for Formula One cars, recognised by the sport's governing body, the Fédération Internationale de l'Automobile (FIA), as the highest class of competition for open-wheel racing cars.\", ' The season started in Australia on 16 March and concluded in Abu Dhabi on 23 November.', \" In the nineteen Grands Prix of the season, a total of eleven teams and twenty-four drivers competed for the World Drivers' and World Constructors' championships.\", ' It was the first Formula One season since to see an accident with ultimately fatal consequences as Jules Bianchi succumbed to the injuries he sustained during the 2014 Japanese Grand Prix.', ' He died on 17 July 2015 after spending nine months in a coma following the accident.']], ['2013 FIA Formula One World Championship', ['The 2013 FIA Formula One World Championship was the 67th season of the FIA Formula One motor racing.', \" It featured the 2013 FIA Formula One World Championship which was open to Formula One cars, recognised by the sport's governing body, the Fédération Internationale de l'Automobile (FIA), as the highest class of competition for open-wheel racing cars.\", \" Eleven teams and twenty-three drivers contest the nineteen Grands Prix that made up the calendar for the 2013 season, with the winning driver being crowned the World Drivers' Champion and the winning team the World Constructors' Champions.\", ' The season started in Australia on 17 March 2013 and ended in Brazil on 24 November 2013.']], ['2020 FIA Formula One World Championship', [\"The 2020 FIA Formula One World Championship is a planned motor racing championship for Formula One cars which would be recognised by the governing body of international motorsport, the Fédération Internationale de l'Automobile (FIA), as the highest class of competition for open-wheel racing cars.\", ' It would be the 74th season of Formula One motor racing and would be the 71st Formula One World Championship.', \" Teams and drivers would be competing for the World Constructors' and World Drivers' championships.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.554\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab3cecd5542992ade7c6eae', 'answer': 'his own opinions changed', 'question': 'How do the Peter Laufer books Forbidden Creatures and No Animals Were Harmed differ in their focus on animals?', 'supporting_facts': [['Forbidden Creatures', 2], ['No Animals Were Harmed (book)', 2], ['No Animals Were Harmed (book)', 3]], 'context': [['Hope Is a Tattered Flag', ['Hope is a Tattered Flag: Voices of Reason and Change for the Post-Bush Era is a 2008 book by Markos Kounalakis and Peter Laufer, with a foreword by Will Durst.', ' The book follows the answers Kounalakis and Laufer get when asking people where they go from here, now that George W. Bush has left office as the President of the United States.', ' The foreword by Will Durst talks about his opinion of the Post-Bush Era.']], ['Acclimatisation society', ['Acclimatisation societies were voluntary associations in the 19th and 20th centuries that encouraged the introduction of non-native species in various places around the world with the hope of their acclimatisation and adaptation.', ' The motivation at the time was a sense that introducing these species of plants and animals would enrich the flora and fauna of a region .', ' These societies were born during a period of colonialism when Europeans began to settle in unfamiliar environments, and the movement sought to establish familiar plants and animals (mainly from Europe) in new areas while also bringing exotic and useful foreign plants and animals into the European centres.', \" Today it is widely understood that introducing species can be harmful to native species and their ecosystems; for example, in Australia plants were harmed by rabbits' overgrazing; in North America house sparrows displace and kill native birds; and around the world, salamander populations are today threatened by introduced fungal infections.\", ' At the time of acclimatisation societies, however, this was insufficiently understood .']], ['Do not feed the animals', ['The prohibition \"do not feed the animals\" reflects a policy forbidding the artificial feeding of wildlife (wild or feral animals) in situations where the animals, or the people doing the feeding, might be harmed.', ' Signs displaying this message are commonly found in zoos, circuses, animal theme parks, aquariums, national parks, parks, public spaces, farms, and other places where people come into contact with wildlife.', ' In some cases there are laws to enforce such no-feeding policies.', ' However, some people (such as some of those who enjoy feeding pigeons in cities) openly and strongly oppose such laws claiming that animals such as pigeons can be an amenity for people who do not have company such as friends or family, and say that the laws prohibiting feeding animals in urban places must change.', ' In some countries, such as Greece, feeding the pigeons in cities is a widespread practice.', ' Cultural hostility to feeding animals in cities and laws that ban the practice raise concerns about how humans relate to other living beings in the urban environment.', ' Politicians have also protested laws that ban feeding feral pigeons in cities.', ' Feral pigeons in cities existed for thousands of years but only recently in some countries humans started seeing them as a nuisance and became hostile to them.', ' In India, feeding feral animals in cities is considered a noble act.', ' Academicians say that how humans treat animals is related to how humans treat each other and thus raise concerns about the cultural shift from seeing feral city pigeons as harmless in the 1800s to seeing them a undesirable in some countries in the 2000s.']], ['American Humane Association', ['American Humane (AH) is an organization founded in 1877, committed to ensuring the safety, welfare and well-being of animals.', \" American Humane's leadership programs are first to serve in promoting and nurturing the bonds between animals and humans.\", ' It was previously called the International Humane Association, before changing its name in 1878.', ' In 1940, it became the sole monitoring body for the humane treatment of animals on the sets of Hollywood films and other broadcast productions.', ' American Humane is best known for its trademarked certification \"No Animals Were Harmed\", which appears at the end of film or television credits.', ' It has also run the Red Star Animal Emergency Services since 1916.', ' In 2000, American Humane formed the Farm Animal Services program, an animal welfare label system for food products.', ' American Humane is currently headquartered in Washington D.C. It is a section 501(c)(3) nonprofit organization.']], ['The Dangerous World of Butterflies', ['The Dangerous World of Butterflies: The Startling Subculture of Criminals, Collectors, and Conservationists is a 2009 book by Doctor of Philosophy Peter Laufer.', ' It is the first book in his untitled animal trilogy, preceding \"Forbidden Creatures\" in 2010 and \"No Animals Were Harmed\" in 2011.', ' The book explores the presence of the butterfly in the fields of organized crime, ecological devastation, species depletion, natural history museum integrity, and chaos theory.']], ['No Animals Were Harmed (book)', ['No Animals Were Harmed: The Controversial Line Between Entertainment and Abuse is a 2011 book by Peter Laufer.', ' It is the third book in his untitled animal trilogy, following \"Forbidden Creatures\" in 2010 and \"The Dangerous World of Butterflies\" in 2009.', ' The book explores what those who work with animals believe to be the line between using animals for entertainment purposes and abusing them.', ' Meanwhile, the author recounts how his own opinions changed about that line when he talks to the different people about their beliefs.']], ['Forbidden Creatures', ['Forbidden Creatures: Inside the World of Animal Smuggling and Exotic Pets is a 2010 book by Doctor of Philosophy Peter Laufer.', ' It is the second book in his untitled animal trilogy, following \"The Dangerous World of Butterflies\" in 2009 and preceding \"No Animals Were Harmed\" in 2011.', ' The book explores the lives of those that either own exotic animals or have been captured for illegally smuggling them, with a strong focus on Travis, the chimpanzee who attacked Charla Nash in 2009.']], ['¡Calexico!', ['¡Calexico!', ' True Lives of the Borderlands is a 2011 book by Doctor of Philosophy Peter Laufer.', \" It covers Laufer's encounters and experiences during his week-long stay in Calexico, California, a city on the Mexico–California border.\", ' He asks citizens there various questions about life on the border, such as what draws them to border towns and if \"English-only\" would be a realistic policy.']], ['List of films about animals', ['This is a list of notable films that are primarily about and/or feature animals.', ' While films involving dinosaurs and other prehistoric animals are included on this list, those concerning mythical creatures, such as dragons or vampires, are not; however, films concerning anthropomorphized animals (such as Scooby-Doo), gigantized animals (such as King Kong), mutated forms of real animals (such as \"Anaconda\"), or fictional hybrids of real animals (such as \"Sharktopus\") are considered to be films about animals, and are thus featured on this list.']], ['Peter Laufer', ['Peter Laufer is an independent American journalist, broadcaster and documentary filmmaker working in traditional and new media.', ' He is the James Wallace Chair in Journalism at the University of Oregon School of Journalism and Communication.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.554\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adf86355542993344016cbd', 'answer': 'Melbourne Storm', 'question': 'Which rugby league team which Rory Kostjasyn played for is based in Melbourne, Victoria?', 'supporting_facts': [['Rory Kostjasyn', 1], ['Melbourne Storm', 0]], 'context': [['Rory Kostjasyn', ['Rory Kostjasyn (born 6 June 1987) is a former Australian professional rugby league footballer.', ' He played in the National Rugby League for the Melbourne Storm and North Queensland Cowboys, with whom he was a member of their 2015 NRL premiership and 2016 World Club Challenge winning sides.', ' An Irish international and New South Wales Country representative, he played at hooker, but could also fill in at lock and five-eighth.']], [\"New South Wales Women's Rugby League\", [\"The New South Wales Women's Rugby League is the governing body of female rugby league in New South Wales.\", \" It is a member of the Australian Women's Rugby League and New South Wales Rugby League.\", \" The organisation is responsible for administering the New South Wales Women's rugby league team, Women's City vs Country Origin, Sydney Metropolitan Women's Rugby League and Country Rugby League Women's RL competitions.\"]], ['David Watkins (rugby)', [\"David Watkins MBE (born 5 March 1942) is a Welsh former dual-code rugby international, having played both rugby union and rugby league football for both codes' national teams between 1963 and 1974.\", ' He captained the British and Irish Lions rugby union side and made six appearances for the Great Britain rugby league team.', ' With the Wales national rugby league team he played in every match of the 1975 World Cup and with English club Salford he played more than 400 games over 12 seasons']], ['Country New South Wales rugby league team', ['The Country New South Wales rugby league team was a representative rugby league football team that consisted of professional players who originated from clubs of the Country Rugby League, one of two federations in the state of New South Wales.', ' Country annually played in the City vs Country Origin competition against the City New South Wales rugby league team, which was made up of players originating from Sydney.']], ['Marika Koroibete', ['Marika Koroibete (born 26 July 1992) is a Fijian-Australian rugby footballer and a dual-code international.', \" He has been capped for Australia's national rugby union team, and currently plays as a winger for the Melbourne Rebels in Super Rugby.\", ' Koroibete previously played rugby league for the Melbourne Storm and Wests Tigers of the National Rugby League, and was a member of the Fijian national rugby league team.']], ['NRL All Stars team', ['The NRL All Stars team was a rugby league football team made up of professional players in the National Rugby League.', ' These players were selected by fan vote.', ' However, fans were only allowed to select one player from each NRL team to join the Australian rugby league team captain and the New Zealand national rugby league team captain.', ' Two other players were selected by the NRL All Stars coach.', ' This team played in the annual NRL All Stars Game against the Indigenous All Stars.', ' They were replaced in 2016 by a World All Stars.']], ['Melbourne Storm', ['Melbourne Storm is a rugby league team based in Melbourne, Victoria, that participates in the National Rugby League.', ' The first fully professional rugby league team based in the state, they entered the competition in 1998.', ' Melbourne Storm was originally a Super League initiative and created in 1997 during the Super League war.', ' The club plays its home games at AAMI Park.', ' The Storm has won three premierships since its inception, in 1999, 2012 and 2017 and has contested several more grand finals.', ' The Storm won two additional premierships, in 2007 and 2009, but these titles were stripped by the NRL following the discovery of a salary cap breach in 2010, which is against the rules of the NRL.', \" Melbourne Storm also competed in the NRL's Under-20s competition (as Melbourne Thunderbolts) from 2008 until its demise in 2017.\", ' In addition, the club has also expanded into netball with a joint venture with University of the Sunshine Coast.', ' The Sunshine Coast Lightning commenced playing in the National Netball League in 2017.']], ['West Wales Raiders Rugby League', ['The West Wales Raiders Rugby League Club previously known as Raiders RL are the most established Rugby league club in West Wales formed in January 2015.', ' They are based at Stebonheath Park in Llanelli.', ' Formerly called Gwendreath Valley Raiders they moved to Llanelli when the club gained entry into the Conference League South which is level 4 of the Rugby League system.', ' in addition to being a local club the raiders are more proactive than some of their professional counterparts by visiting all the local schools delivering rugby league training to pupils.', ' They also have links with one of the oldest university rugby league teams in Wales the Warpigs from Swansea University.', ' The raiders have taken over the running and coaching of the university rugby league team since 2017.']], ['Fiji national rugby league team', ['The Fiji national rugby league team, nicknamed the Bati (pronounced ] ), has been participating in international rugby league football since 1992.', ' The team is controlled by the governing body for rugby league in Fiji, Fiji National Rugby League (FNRL), which is currently a member of the Asia-Pacific Rugby League Confederation (APRLC).', \" Fiji have come within one victory of the Rugby League World Cup Final twice (in 2008 and 2013) and are currently ranked 8th in the Rugby League International Federation's World Rankings.\"]], ['Mike Nicholas', ['Mike Nicholas is a former rugby league and rugby union footballer who played in the 1970s, and 1980s, and coach or team manager of the Wales Rugby League team since the 1980s, he is currently President of Wales Rugby League.', ' He played club level rugby union for Aberavon.', ' In rugby league he played for Warrington and the Cardiff Blue Dragons, and at representative Rugby League level for Wales and Great Britain.', ' He played as a Front Row forward or Second Row, i.e. numbers 8, 10, 11 or 12.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.555\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac0d9a35542992a796ded90', 'answer': 'no', 'question': 'Are Hungry Hungry Hippos and Parcheesi both published by Parker Brothers?', 'supporting_facts': [['Hungry Hungry Hippos', 0], ['Parcheesi', 0]], 'context': [['Hungry Shark', ['Hungry Shark is a series of mobile games developed/published by Future Games of London (prior to Hungry Shark Evolution) and published by Ubisoft (since Hungry Shark Evolution).', ' The games allow players to control several unique species of sharks, including mako sharks, great white sharks, hammerhead sharks, reef sharks, and megalodon; to progress, the player must consume other marine animals and grow in size until the next, more powerful shark is available for purchase.', ' In May 2016, Hungry Shark World was downloaded 10 million times in six days, reaching the top 10 free iPhone and Android apps.']], ['Lord of the Rings: Journey to Rivendell', ['Lord of the Rings: Journey to Rivendell was a video game scheduled to be released for the Atari 2600 and published by Parker Brothers, having been announced in their 1983 catalog.', \" However, it was never released, and it was believed little or no work done on the game's coding.\", ' Twenty years later, a former Parker Brothers employee gave a prototype of the game to the operator of the AtariAge website.', ' Surprisingly, the game was quite complex and seemed to be complete.', ' Several characters from the book make appearances in the game, including Frodo Baggins, Sam Gamgee, Aragorn, Gandalf, Tom Bombadil, and Glorfindel.']], ['Scattergories', ['Scattergories is a creative-thinking category-based party game originally published by Parker Brothers in 1988.', ' Parker Brothers was purchased by Hasbro a few years later, and they published the game internationally under their Milton Bradley brand.', ' The objective of the 2-to-6-player game is to score points by uniquely naming objects within a set of categories, given an initial letter, within a time limit.', ' The game is based on a traditional game known as Tutti Frutti, Jeu du Baccalauréat, Stadt Land Fluss, and many other names.']], ['Hungry Hungry Hippos', ['Hungry Hungry Hippos is a tabletop game made for 2–4 players, produced by Hasbro, under the brand of its subsidiary, Milton Bradley.', ' The idea for the game was published in 1967 by toy inventor Fred Kroll and it was introduced in 1978.', \" The objective of the game is for each player to collect as many marbles as possible with their 'hippo' (a toy hippo model).\", ' The game is marketed under the \"Elefun and Friends\" banner, along with \"Elefun\", \"Mouse Trap\" and \"Gator Golf\".', ' The game was referenced in the 2010 Disney Pixar movie, \"Toy Story 3\" and the 2001 cult film \"Donnie Darko\".', ' There is also a battle level based on the game in the 2016 Micro Machines game.']], ['Parker Bros.', ['Parker Bros., also known at various times as Parker Brothers Manufacturing Company, Parker Brothers Guns, and Parker Bros.', ' Shotguns, was an American firm almost exclusively producing shotguns from 1867-1942.', ' During these years, approximately 242,000 guns were produced in various grades, and are widely considered the finest and most collectible American shotgun.']], ['Parcheesi', ['Parcheesi is a brand-name American adaptation of the Indian cross and circle board game Pachisi, published by Parker Brothers and Winning Moves.']], ['Parker Brothers', ['Parker Brothers was an American toy and game manufacturer which later became a brand of Hasbro.', ' More than 1,800 games were published under the Parker Brothers name since 1883.', ' Among its products were \"Monopoly\", \"Cluedo\" (licensed from the British publisher and known as \"Clue\" in North America), \"Sorry!', '\", \"Risk\", \"Trivial Pursuit\", \"Ouija\", \"Aggravation\", \"Bop It\" and \"Probe\".', ' The trade name is now defunct; former products are marketed under the \"Hasbro Gaming\" label.']], ['Star Wars: Return of the Jedi: Ewok Adventure', ['Star Wars: Return of the Jedi: Ewok Adventure, also known as Revenge of the Jedi: Game I, is a cancelled 1983 shoot \\'em up video game based on the 1983 \"Star Wars\" film \"Return of the Jedi\".', ' The game was developed by Atari Games and was to be published by Parker Brothers on the Atari 2600.', ' Although it was completed, the game was never released for sale, as the marketing department of Parker Brothers considered the controls too difficult to master.', ' A prototype cartridge surfaced in 1997.']], ['Hungry: A Mother and Daughter Fight Anorexia', ['Hungry: A Mother and Daughter Fight Anorexia is a 2009 book cowritten by Sheila Himmel and Lisa Himmel.', ' Written by a daughter and her mother, \"Hungry\" depicts Lisa Himmel\\'s struggle with anorexia and bulimia.', ' Published by Berkley Trade, \"Hungry\" took six years to be completed because of Lisa\\'s relapses.']], ['Masterpiece (game)', ['Masterpiece is a board game by Parker Brothers, now a brand of Hasbro.', ' Players participate in auctions for famous works of art.', ' It was invented by Joseph M. Burck of Marvin Glass and Associates and originally published in 1970 by Parker Brothers, and then published again in 1976 and 1996.', ' The game is now out-of-print.', ' In this game, players compete with other players to bid on potentially valuable paintings, and negotiate with other players to trade these works of art, build a portfolio, amass money, and win the game.', ' The top value of a painting in the 1970 edition is $1 million, and $10 million in the 1996 edition; however, getting the full value for the painting requires some luck in landing on the right square on the board to sell a painting to the bank.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.556\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7d7d895542995f4f402281', 'answer': 'score 1,000 points with two different teams', 'question': 'As the oldest player in the NFL, this football player is the only player ever to have accomplished what?', 'supporting_facts': [['Phil Dawson', 3], ['Adam Vinatieri', 5]], 'context': [['Jim Brown', ['James Nathaniel Brown (born February 17, 1936) is a former professional American football player and actor.', ' He was a fullback for the Cleveland Browns of the National Football League (NFL) from 1957 through 1965.', ' Considered to be among the greatest football players of all time, Brown was a Pro Bowl invitee every season he was in the league, was recognized as the AP NFL Most Valuable Player three times, and won an NFL championship with the Browns in 1964.', ' He led the league in rushing yards in eight out of his nine seasons, and by the time he retired, he had shattered most major rushing records.', ' In 2002, he was named by \"The Sporting News\" as the greatest professional football player ever.']], ['Ned Hughes', ['Edward \"Ned\" Hughes (26 April 1881 – 1 May 1928), was a New Zealand rugby union and rugby league player who played 9 times (6 of these were test matches) as an All Black hooker from 1907 until 1921 and twice for the Kiwis in 1910.', ' His All Black career is unique in that there was a gap of 13 years between test matches, and that he is the oldest player ever to have played for the All Blacks, at age 40 years, 123 days.']], ['Mark Recchi', ['Mark Louis Recchi (born February 1, 1968) is a Canadian ice hockey coach and former professional player.', ' He is currently an assistant coach for the Pittsburgh Penguins Recchi played 22 years in the National Hockey League.', ' Recchi was a member of three Stanley Cup-winning teams, the 1991 Pittsburgh Penguins, the 2006 Carolina Hurricanes and the 2011 Boston Bruins.', ' In Game Two of the 2011 Finals, he became the oldest player ever to score in a Stanley Cup Finals series.']], ['Adam Vinatieri', ['Adam Matthew Vinatieri (born December 28, 1972) is an American football placekicker for the Indianapolis Colts of the National Football League (NFL).', ' He has played in five Super Bowls: four with the New England Patriots and one with the Colts.', ' Vinatieri won Super Bowls in 2001, 2003, and 2004 with the Patriots, as well as in 2006 with the Colts.', ' Among placekickers, he holds NFL records for most Super Bowl appearances (5) and most Super Bowl wins (4).', ' He also holds NFL records, among all players, for most postseason points scored (234), and most overtime field goals made (10).', ' He is the only player ever to score 1,000 points with two different teams.', ' As of the 2016 season, Vinatieri, 44, is the oldest active player in the NFL.', ' Vinatieri has converted the 3rd most field-goals in NFL history (529) as well as attempted the 4th most field-goals in NFL history (627), 12 behind George Blanda.']], ['Maurice Roberts', ['Maurice \"Moe\" Roberts (December 13, 1905 – February 7, 1975) was an American ice hockey player, who was the oldest man ever to play the position of goaltender in National Hockey League history, and in two different stretches of several decades was both the oldest player ever to play a NHL game and the \"youngest\" man ever to play goal in the NHL.']], ['Giuseppe Romano', ['Giuseppe Romano (November 15, 1918 – November 16, 1965) was an Italian professional football player.', ' Born in Brescia, he was the oldest player ever to play for Juventus F.C., at 38 years, 138 days of age.', ' He was born Brescia, Italy and died in Tempio Pausania, Italy.']], ['Phil Dawson', ['Philip Drury Dawson (born January 23, 1975) is an American football placekicker for the Arizona Cardinals of the National Football League (NFL).', ' He played for the Cleveland Browns from 1999 to 2012 and holds their franchise record for most field goals made, passing Hall of Famer Lou Groza in 2010.', ' He played college football at Texas.', ' As of the end of the 2016 NFL season, Dawson is the second oldest player in the league, behind fellow placekicker Adam Vinatieri.']], ['Lyn Carpenter', ['Lyn Carpenter is an England Netball volunteer administrator and former National representative player.', ' Carpenter was the oldest player ever to be awarded a debut international cap in the England national netball team, which she received in December 1997 at the age of 32.', ' During her senior international career she amassed 33 international caps, winning bronze medals at the 1998 Commonwealth Games in Kuala Lumpur and the 1999 Netball World Championships in New Zealand.', ' She also represented Great Britain in basketball at the 1987 World Student Games in Zagreb.', ' Carpenter served as Vice Chairman of England Netball until August 2014 when she left the organisation abruptly.', ' She is the current Chairman of Netball Europe.', ' Since 2009, Carpenter has worked at Hammersmith Council as Director of Residents Services.', ' In September 2011 she was appointed to a new Biborough Executive Director role that also includes the Royal Borough managing a range of complex universal services.']], ['Jakov Surać', ['Jakov Surać (born 12 February 1975) is a Croatian football midfielder, playing for NK Zadar in the Prva HNL.', ' In July 2014 he set a record of being the oldest player ever to play in a Prva HNL match.']], ['Alvin Wistert', ['Alvin Lawrence \"Moose\" Wistert (June 26, 1916 – October 3, 2005) was an American football player.', ' A native of Chicago, Illinois, he played college football at the tackle position for Boston University in 1946 and at the University of Michigan from 1947 to 1949.', ' He began his collegiate football career at age 30 following 12 years of working in a factory and serving in the United States Marine Corps during World War II.', ' He played at the defensive tackle position for the undefeated 1947 and 1948 Michigan Wolverines football teams, both of which finished the season ranked No. 1 in the final Associated Press poll.', ' He also holds the distinction of being the oldest college football player ever selected as a College Football All-American, having been selected to the 1948 College Football All-America Team at age 32 and the 1949 Team at age 33.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.557\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab48c325542996a3a969f93', 'answer': 'Essex', 'question': 'In which county is the English king Harold Godwinson buried?', 'supporting_facts': [['Edith Walks', 0], ['Waltham Abbey Church', 0]], 'context': [['Tostig Godwinson', ['Tostig Godwinson ( 1026 – 25 September 1066) was an Anglo-Saxon Earl of Northumbria and brother of King Harold Godwinson.', \" After being exiled by his brother, Tostig supported the Norwegian king Harald Hardrada's invasion of England, and was killed at the Battle of Stamford Bridge.\"]], ['Edith the Fair', ['Edith the Fair (Old English: \"Ealdgȳð Swann hnesce\" , \"Edyth the Gentle Swan\"; c. 1025 – c. 1086), also known as Edith Swanneck, was the first wife of King Harold Godwinson.', ' \"Swanneck\" (or Swan-Neck) comes from the folk etymology which made her in Old English as \"swann hnecca\", \"swan neck\", which was actually most likely a corrupted form of \"swann hnesce, \"\"Gentle Swan\"\" \".', \" She is sometimes confused with Ealdgyth, daughter of Earl Ælfgar of Mercia, who was queen during Harold's reign.\"]], ['Edith Walks', ['Edith Walks is a 2017 documentary film directed by Andrew Kötting which imagines a journey by Edith the Fair, wife of English king Harold Godwinson, from Waltham Abbey where he is buried to near the site of the Battle of Hastings and the invasion of England by William the Conqueror in 1066.', ' It includes contributions from the writers Alan Moore and Iain Sinclair, the torch singer Claudia Barton, and the musician Jem Finer.']], ['Cultural depictions of Harold Godwinson', ['Fictional accounts based on the events surrounding Harold Godwinson\\'s brief reign as king of England have been published, notably the play \"Harold\", by Alfred, Lord Tennyson, in 1876; and the novel \"Last of the Saxon Kings\", by Edward Bulwer-Lytton, in 1848.', ' Rudyard Kipling wrote a short story, included in his 1910 collection, \"Rewards and Fairies\", where an aged King Harold meets Henry I and dies in the arms of a Saxon knight.']], ['The Last English King', ['The Last English King (1997) is a historical novel by English writer Julian Rathbone.', ' The novel covers the time of the Battle of Hastings.', ' It revolves around Walt Edwinson, a housecarl of Harold Godwinson, the last Anglo-Saxon king of England.', ' The story starts with Walt returning to his home at Iwerne in Dorset four years after the Battle of Hastings.', ' He had fled England after the defeat of the Anglo-Saxons and had spent the time travelling across Europe and Asia Minor.', \" The story of his journey from Constantinople via Nicomedia and Nicaea to Side is then recounted in parallel with his recollections of the time before the battle, such as his accompanying Harold to William of Normandy's attack on Dinan.\"]], ['Leofwine Godwinson', ['Leofwine Godwinson (c. 1035 – 14 October 1066) was a younger brother of King Harold Godwinson, the fifth son of Earl Godwin.']], ['Battle of Stamford Bridge', [\"The Battle of Stamford Bridge took place at the village of Stamford Bridge, East Riding of Yorkshire, in England on 25 September 1066, between an English army under King Harold Godwinson and an invading Norwegian force led by King Harald Hardrada and the English king's brother Tostig Godwinson.\", ' After a bloody battle, both Hardrada and Tostig along with most of the Norwegians were killed.', ' Although Harold Godwinson repelled the Norwegian invaders, his army was defeated by the Normans at Hastings less than three weeks later.', ' The battle has traditionally been presented as symbolising the end of the Viking Age, although major Scandinavian campaigns in Britain and Ireland occurred in the following decades, such as those of King Sweyn Estrithson of Denmark in 1069–1070 and King Magnus Barefoot of Norway in 1098 and 1102–1103.']], ['Gytha Thorkelsdóttir', ['Gytha Thorkelsdóttir (Old English: \"Gȳða Þorkelsdōttir\" , 997 – c. 1069), also called Githa, was a Danish noblewoman.', ' She was the mother of King Harold Godwinson and of Edith of Wessex, queen consort of King Edward the Confessor of England.']], ['Waltham Abbey Church', ['The Abbey Church of Waltham Holy Cross and St Lawrence is the parish church of the town of Waltham Abbey, Essex, England.', ' It has been a place of worship since the 7th century.', ' The present building dates mainly from the early 12th century and is an example of Norman architecture.', ' To the east of the existing church are traces of an enormous eastward enlargement of the building, begun following the re-foundation of the abbey in 1177.', ' In the Late Middle Ages, Waltham was one of the largest church buildings in England and a major site of pilgrimage; in 1540 was the last religious community to be closed during the Dissolution of the Monasteries.', ' It is still an active parish church for the town.']], ['Godwin, Earl of Wessex', ['Godwin of Wessex (Old English: \"Godƿin\" ; 100115 April 1053) was one of the most powerful earls in England under the Danish king Cnut the Great and his successors.', ' Cnut made him the first Earl of Wessex.', ' Godwin was the father of King Harold Godwinson and Edith of Wessex, wife of King Edward the Confessor.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.558\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae3a3835542990afbd1e19c', 'answer': 'Kyle Busch', 'question': 'The driver know for doing backflips off his car lost to which driver in the 2009 NASCAR Nationwide Series?', 'supporting_facts': [['2009 NASCAR Nationwide Series', 2], ['2009 NASCAR Nationwide Series', 3], ['Carl Edwards', 4]], 'context': [['Justin Allgaier', ['Justin Allgaier (born June 6, 1986) is an American professional stock car racing driver.', ' He currently competes full-time in the NASCAR Xfinity Series, driving the No. 7 Chevrolet Camaro for JR Motorsports.', ' He was the 2008 ARCA Re/MAX Series Champion and the 2009 NASCAR Nationwide Series Rookie of the Year.']], ['Diamond Ridge Motorsports', ['No.', ' Pastrana 199 Racing (formerly Diamond Ridge Motorsports, Diamond-Waltrip Racing, and Pastrana-Waltrip Racing) is a NASCAR Nationwide Series team that competed in the NASCAR Cup Series and Busch Series from 1990 to 1999, and revived as a Nationwide Series team in 2010.', ' The original Diamond Ridge team was owned and operated by Gary Bechtel.', ' Despite modest success in the Busch Series, the team was never able to maintain a competitive level in the Winston Cup Series.', ' The team was revived as a partnership with Michael Waltrip Racing in 2010 as Diamond-Waltrip Racing, running full-time in the Nationwide Series with Trevor Bayne.', \" For 2011 the team partnered with action star Travis Pastrana and MWR development driver Ryan Truex, though the team shuttered temporarily due to Pastrana's injuries at the 2011 Summer X Games and a lacking sponsor.\", \" The team changed its name to Pastrana 199 Racing, a reference to Pastrana's standard number.\", ' The team was to field the No. 99 for Pastrana, but the deal was cancelled when Michael Waltrip Racing aligned with RAB Racing to field the No. 99.']], ['Alex García (racing driver)', ['Alex García (born March 18, 1977) is a Venezuelan stock car racing driver, and a former competitor in the NASCAR Nationwide Series.', ' A road course ringer, he drove the 98 Dixien/OmniSource Chevrolet for Transnet Racing, a team which García owns.', \" Alex García made his NASCAR Nationwide Series debut in the 2007 Telcel Motorola Mexico 200 in Mexico City, where he became the first Venezuelan to race in one of NASCAR's top three series.\"]], ['Kyle Busch', ['Kyle Thomas \"Rowdy\" Busch (born May 2, 1985) is an American professional stock car racing driver and team owner.', ' He is the driver of the No. 18 Toyota Camry in the Monster Energy NASCAR Cup Series and the No. 18 Camry in the NASCAR Xfinity Series for Joe Gibbs Racing.', ' He also owns Kyle Busch Motorsports, which runs multiple trucks in the Camping World Truck Series.', ' Busch is the 2009 NASCAR Nationwide Series champion and the 2015 NASCAR Sprint Cup Series champion.']], ['2009 NASCAR Nationwide Series', ['The 2009 NASCAR Nationwide Series began on February 14 at Daytona International Speedway with the Camping World 300.', ' The season concluded on November 21 at Homestead-Miami Speedway with the Ford 300.', ' With 25 top-five finishes, Kyle Busch was the season champion.', ' He finished 210 points clear of Carl Edwards and 318 ahead of Brad Keselowski.']], ['Justin Hobgood', ['Justin Hobgood (born July 31, 1979 in Winnsboro, South Carolina) is an American race car driver in the NASCAR Nationwide Series and Camping World Truck Series.', ' Chapman drives the #91 Chevy Monte Carlo part-time for MSRP Motorsports, a start and park team in the Nationwide series.', \" His career best finish (27th) came in his first start in the 2003 Sam's Town 250 on October 18.\", ' He has made a handful of starts In 2003, 2008 and 2009 in the Nationwide Series.', ' He also has a few truck starts; his best finish and first ever NASCAR top-10 came at Talladega Superspeedway in the fall where he ended up 9th in a very wild race.']], ['2013 DRIVE4COPD 300', ['The 2013 DRIVE4COPD 300 was a NASCAR Nationwide Series race held on February 23, 2013 at Daytona International Speedway in Daytona Beach, Florida.', ' It was the first race of the 2013 NASCAR Nationwide Series season.', \" The race was the 32nd running of the event, and the pole position given to Roush Fenway Racing's Trevor Bayne with a lap speed of 177.162 mph , while Tony Stewart of Richard Childress Racing won the race.\", ' Sam Hornish, Jr. finished 2nd and Alex Bowman finished 3rd.']], ['Dwayne Leik', ['Dwayne Leik (born February 9, 1964) is a former NASCAR driver.', ' He was formerly an owner of Leik Motorsports and a part-time driver in the NASCAR Nationwide Series.', ' He formerly worked as the business manager for Marcis Racing and legendary NASCAR driver Dave Marcis.', ' Leik was able to secure racing sponsorship for Marcis for eleven consecutive seasons and he is widely regarded as the key figure in resurrecting Marcis’ faltering career.', \" Since Marcis Racing did not have a test team, Leik's car was often seen as a test car for Marcis Racing.\", ' Leik also accompanied Marcis and participated in the IROC (International Race of Champions) test sessions.', ' He once logged over 1,100 mi in an IROC car at Daytona in a single day test.', ' He has also served as a driving instructor at the Fast Track High Performance Driving School and Richard Petty Driving School.']], ['2013 NASCAR Nationwide Series', ['The 2013 NASCAR Nationwide Series was the 32nd season of the Nationwide Series, a stock car racing series sanctioned by the NASCAR in the United States.', ' The season was contested over thirty-three races and started on February 23, 2013 at Daytona International Speedway, with the DRIVE4COPD 300, and ended on November 16 with the Ford EcoBoost 300 at Homestead-Miami Speedway.', \" Austin Dillon of Richard Childress Racing won the Drivers' Championship, becoming the first driver in the three major NASCAR series to do so without recording a win.\", \" The No. 22 entry of Penske Racing won the Owners' Championship, while Ford won the Manufacturers' Championship.\"]], ['Carl Edwards', ['Carl Michael Edwards II (born August 15, 1979) is a retired American professional stock car racing driver.', ' He competed in the NASCAR Sprint Cup Series, driving the No. 19 Toyota Camry for Joe Gibbs Racing.', ' Prior to that, he drove the No. 99 Ford Fusion for Roush Fenway Racing.', ' He won the 2007 NASCAR Busch Series championship and nearly won the 2011 NASCAR Sprint Cup Series title, but lost by a tiebreaker to Tony Stewart.', ' Edwards is well known for doing a backflip off of his car to celebrate a victory, which was a result of saving himself from a potential fall when he had his first win.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.559\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a89217455429938390d4039', 'answer': 'small family car', 'question': 'What is the British acceptation of the type of sedan the Maruti Suzuki Dzire is? ', 'supporting_facts': [['Maruti Suzuki Dzire', 0], ['Maruti Suzuki Dzire', 1], ['Compact car', 0]], 'context': [['Maruti Brezza', ['The Maruti Suzuki Vitara Brezza is a compact SUV unveiled in 2016 at the 13th Auto Expo 2016.', \" It was the company's first attempt in sub-compact SUV space and fourth attempt in the SUV market after Gypsy, Grand Vitara and S-Cross.\", 'The Brezza is the first car which was conceptualized and designed in India by Maruti Suzuki.', \" The Brezza was designed by C.V Raman, Maruti's head of design and all parts of its production were developed in India.\", ' A consortium of 15 auto journalists declared Vitara Brezza the \"Indian Car of the Year (ICOTY) 2017\" among all the cars launched in 2016.', ' It battled it out against the Hyundai Tucson and Toyota Innova Crysta.']], ['Maruti Eeco', ['The Maruti Eeco was introduced in India by Maruti Suzuki during early 2010.', ' This car is essentially a stripped down version of defunct Versa.', ' This is an urgent refresh in lines with Maruti Suzuki not being able to come up with a new minivan.', ' EECO comes with 5-seater and 7-seater options.', ' Eeco is equipped with advanced Engine Management System for optimizing fuel efficiency and performance.', ' It is branded by the manufacturer as \"a perfect car for every occasion\"-a business trip or a picnic with the loved ones.']], ['Suzuki Motorcycle India Limited', ['Suzuki Motorcycle India, Private Limited (SMI) is the wholly owned Indian subsidiary of Suzuki, Japan.it was the third Suzuki automotive venture in India, after TVS Suzuki(1982-2001) and Maruti Suzuki(1982).', 'In 1982 the joint-venture between Suzuki Motor Corporation and TVS Motor Company incorporated and started production of two wheelar in india.', ' In 2001, after separating ways with TVS motor company, the company was re entered as Suzuki Motorcycle India , Private Limited (SMI) in 2006 ,The company has set up a manufacturing facility at Gurgaon, Haryana having the annual capacity of 5,40,000 units.']], ['Maruti Suzuki True Value', ['Maruti Suzuki True Value is the first major automobile OEM to enter India’s used car market in 2001, Maruti Suzuki True Value is the pre-owned cars arm of Maruti Suzuki that offers buying, selling and exchange of certified pre-owned cars for customers.', ' Maruti Suzuki True Value also provides services like Finance, insurance, as well as accessories, through a countrywide network spanning 1,132 outlets across 880 cities.']], ['Maruti Suzuki', ['Maruti Suzuki India Limited, formerly known as Maruti Udyog Limited, is an automobile manufacturer in India.', ' It is a 56.21%-owned subsidiary of Japanese automobile and motorcycle manufacturer Suzuki Motor Corporation.', ' s of 2017 , it had a market share of 51% of the Indian passenger car market.', ' Maruti Suzuki manufactures and sells popular cars such as the Ciaz, Ertiga, Wagon R, Alto, Swift, Celerio, Swift Dzire, Omni, Baleno and Baleno RS.', ' The company is headquartered at New Delhi.', ' In February 2012, the company sold its ten millionth vehicle in India.']], ['Suzuki F10D engine', ['Suzuki F10D engine is an inline 4-cylinder 1061cc engine that was developed in India by Maruti Suzuki for the domestic market.', ' It was debuted in the Maruti Wagon-R in India in 2001.', ' It was briefly installed in Maruti Alto and it was the engine that the first Maruti Zen Estilo came with.', ' This engine is very similar to the 3-cylinder F8D 12-valve engine that was optional on the Maruti 800 at the time.', ' The bore and stroke of F10D is the same as that of the smaller sibling and shares quite a few parts like pistons, rings, conrods and valves.', ' The cast-iron engine block is very similar to that of the older F10A 970cc engine that powered the earlier Maruti Gypsy and Maruti 1000.', ' Both F10A and F10D shares the same stroke length; but interchangeability of parts between these two engines is not known.']], ['Suzuki MR Wagon', ['The Suzuki MR Wagon is a 4-seater mini MPV manufactured by Suzuki for the Japanese market only, and also marketed in Japan by Nissan as the Nissan Moco under an OEM agreement.', ' The model debuted in 2001, and since 2011 it has been in its third generation.', ' It was launched in India by Maruti Suzuki as Maruti Zen Estilo in 2006, Maruti Zen Estilo was discontinued in 2009 and renamed as Maruti Estilo.']], ['Compact car', ['A compact car (North America), or small family car in British acceptation, is a classification of cars that are larger than a subcompact car but smaller than a mid-size car, roughly equivalent to the C-segment in Europe.']], ['Maruti Suzuki Dzire', ['The Maruti Suzuki Dzire (earlier known as Maruti Suzuki Swift Dzire) is a subcompact sedan that has been developed as an extension of the Swift hatchback.', ' The compact sedan was introduced in India back in 2008.', ' It is available in a total of 14 variants including both petrol as well as diesel engine trims.']], ['Suzuki Cultus Crescent', ['The Suzuki Cultus Crescent is a compact car that was produced by Suzuki in Japan between 1995 and 2002, with South Asian production continuing until 2007.', ' The Cultus Crescent was sold as such in Japan until May 1998, when it was renamed Suzuki Cultus due to the sales discontinuation of the previous Cultus in the Japanese market.', ' The Cultus Crescent was also marketed as the Suzuki Esteem in North America, and as the Suzuki Baleno (Japanese: スズキ・バレノ , Suzuki Bareno ) throughout Asia, Australasia, Europe and South America.', ' In India where it was manufactured by Maruti Suzuki, the Cultus Crescent was sold as the Maruti Baleno.', ' In the Philippines, it was marketed as the Chevrolet Cassia.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.559\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a84a7f95542992a431d1a85', 'answer': 'Julia Compton Moore', 'question': 'Who was the wife of the United States Army lieutenant general who received the Distinguished Service Cross, and was the first of his West Point class to be promoted to brigadier general, major general, and lieutenant general', 'supporting_facts': [['Julia Compton Moore', 0], ['Julia Compton Moore', 1], ['Hal Moore', 0], ['Hal Moore', 1]], 'context': [['Charles Pede', ['Lieutenant General Charles N. Pede is a military lawyer who currently serves as the 40th Judge Advocate General of the United States Army.', ' General Pede was promoted from the rank of Brigadier General to the rank of Lieutenant General, bypassing the rank of Major General on July 26, 2017.']], ['Andrew Davis Bruce', ['Lieutenant General Andrew Davis Bruce (September 14, 1894 – July 28, 1969) was an American academic and soldier who served as the third president of the University of Houston.', ' He retired from the United States Army in 1954 as a lieutenant general after seeing action in both World War I and World War II and founding Fort Hood, Texas.', \" Three countries, France, the Philippines, and the United States, awarded him service medals, including the Distinguished Service Cross, the U.S. Army's second highest military decoration.\", ' Bruce is interred in Arlington National Cemetery.']], ['Distinguished Service Cross (United States)', ['The Distinguished Service Cross is the second highest military award that can be given to a member of the United States Army (and previously, the United States Army Air Forces and the United States Air Force), for extreme gallantry and risk of life in actual combat with an armed enemy force.', ' Actions that merit the Distinguished Service Cross must be of such a high degree that they are above those required for all other U.S. combat decorations but do not meet the criteria for the Medal of Honor.', ' The Distinguished Service Cross is equivalent to the Navy Cross (Navy and Marine Corps), the Air Force Cross (Air Force), and the Coast Guard Cross (Coast Guard).']], ['Richard E. Cavazos', [\"Richard Edward Cavazos (born January 31, 1929), a Korean War recipient of the Distinguished Service Cross as a first lieutenant, who advanced in rank to become the United States Army's first Hispanic four-star general.\", ' During the Vietnam War, as a lieutenant colonel, Cavazos was awarded a second Distinguished Service Cross.', ' In 1976, Cavazos became the first Mexican American to reach the rank of brigadier general in the U.S. Army.', ' Cavazos served with great distinction for thirty-three years, with his final command as head of the U.S. Army Forces Command.']], ['Major general (United States)', ['In the United States Army, United States Marine Corps, and United States Air Force, major general is a two-star general-officer rank, with the pay grade of O-8.', ' Major general ranks above brigadier general and below lieutenant general.', ' A major general typically commands division-sized units of 10,000 to 15,000 soldiers.', ' Major general is equivalent to the two-star rank of rear admiral in the United States Navy and United States Coast Guard and is the highest permanent rank during peacetime in the uniformed services.', ' (The higher ranks are temporary ranks linked to specific positions, although virtually all officers who have been promoted to those ranks are approved to retire at their highest earned rank.)']], ['Nadja West', ['Nadja Y. West (born 1961) is a United States Army lieutenant general and the 44th U.S. Army Surgeon General and Commanding General of the U.S. Army Medical Command.', ' West is the first black Army Surgeon General, and was the first black female, active-duty, major general and the first black female major general in Army Medicine.', ' West is also the first Army black female lieutenant general.', ' She is the highest ranking female to have graduated from the United States Military Academy.', ' She received the Army Distinguished Service Medal, the Defense Superior Service Medal, the Legion of Merit, and others.']], ['Frank A. Armstrong', ['Frank Alton Armstrong Jr. (May 24, 1902 – August 20, 1969) was a lieutenant general of the United States Air Force.', ' As a brigadier general in the United States Army Air Forces during World War II, he was the inspiration for the main character in the novel and subsequent film, \"Twelve O\\'Clock High.\"', ' After the war, he held a variety of senior leadership positions prior to and following the establishment of the USAF as an independent service in 1947.', ' Promoted to major general in 1950, he advanced to lieutenant general in 1956 and retired at that rank in 1962.']], ['Hal Moore', ['Harold Gregory \"Hal\" Moore, Jr. (February 13, 1922 – February 10, 2017) was a United States Army lieutenant general and author.', \" He was a recipient of the Distinguished Service Cross, which is the U.S. military's second highest decoration for valor, and was the first of his West Point class (1945) to be promoted to brigadier general, major general, and lieutenant general.\"]], ['Wayne W. Lambert', ['Wayne W. Lambert (born 1936) was an Air force Brigadier General (United States).', ' He attended the United States Military Academy at West Point, New York, after already having served a year in the US Army.', ' Lambert graduated from the Academy in 1959 and was commissioned a 2nd Lieutenant in the USAF that same year.', ' He trained as a bomber pilot, receiving his Aviator badge in 1960.', ' He has logged over 5,800 flying hours with the US Air Force.', ' A seasoned combat veteran, Lambert flew over 225 combat missions in South East Asia as a B-52 crew member.', ' He was awarded the Distinguished Flying Cross (United States) and received 10 awards of the Air Medal for his service during the Vietnamese War.', ' After serving in a succession of lesser command positions, Lambert was promoted to the rank of Brigadier General in 1983, assuming command of the former Strategic Air Command’s 7th Air Division, headquartered then at Ramstein Air Base in West Germany, and overseeing SAC operations throughout the European theater.', ' The Brigadier General retired from the US Air Force in 1989.', ' Lambert also holds an MBA from Chapman College (’76).', ' In retirement, Lambert publicly supported U.S. presidential candidate Mitt Romney and U.S. vice presidential candidate Paul Ryan in the 2012 general election.']], ['Julia Compton Moore', ['Julia Compton Moore (February 10, 1929April 18, 2004) was the wife of Lieutenant General (Ret.)', ' Hal Moore, a United States Army officer.', ' Her efforts and complaints in the aftermath of the Battle of Ia Drang prompted the U.S. Army to set up survivor support networks and casualty notification teams consisting of uniformed officers, which are still in use.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.560\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab4b2935542996a3a969fb5', 'answer': 'Death in the Afternoon', 'question': 'Between Death in the Afternoon and Hennchata, whose inventor has more literary fame?', 'supporting_facts': [['Death in the Afternoon (cocktail)', 1], ['Hennchata', 1]], 'context': [['Lucia Berlin', ['Lucia Brown Berlin (November 12, 1936 – November 12, 2004) was an American short story writer.', ' She had a small, devoted following, but did not reach a mass audience during her lifetime.', ' She rose to sudden literary fame eleven years after her death, in August 2015, with Farrar, Straus and Giroux\\'s publication of a volume of selected stories, \"A Manual For Cleaning Women\", edited by Stephen Emerson.', ' It hit \"The New York Times\" bestseller list in its second week, and within a few weeks, had outsold all her previous books combined.', ' The collection was ineligible for most of the year-end awards (either because she was deceased, or it was recollected material), but was named to a large number of year-end lists, including the New York Times Book Review\\'s \"10 Best Books of 2015.\"', ' It was also a finalist for the Kirkus Prize.']], ['Ion Agârbiceanu', ['Ion Agârbiceanu (September 12, 1882 – May 28, 1963) was an Austro-Hungarian-born Romanian writer, journalist, politician, theologian and Greek-Catholic priest.', ' A native of Transylvania, he graduated from Budapest University, after which he was ordained.', ' He was initially assigned to a parish in the Apuseni Mountains, which form the backdrop to much of his fiction.', ' Before 1910, Agârbiceanu had achieved literary fame in both Transylvania and the Kingdom of Romania; his work was disputed between the rival schools of \"Sămănătorul\" and Poporanism.']], ['Tao Yuanming', ['Tao Yuanming (365?', \"–427), also known as Tao Qian (Hanyu Pinyin) or T'ao Ch'ien (Wade-Giles), was a Chinese poet who lived during the Eastern Jin (317-420) and Liu Song (420-479) dynasties.\", ' He is considered to be one of the greatest poets of the Six dynasties period.', ' Tao Yuanming spent most of his life in reclusion, living in a small house in the countryside, reading, drinking wine, receiving the occasional guest, and writing poems in which he often reflected on the pleasures and difficulties of life in the countryside, as well as his decision to withdraw from civil service.', ' His simple, direct, and unmannered style was at odds with the norms for literary writing in his time.', ' Although he was relatively well-known as a recluse poet in the Tang dynasty (618-907), it was not until the Northern Song dynasty (960-1127), when influential literati figures such as Su Shi (1037-1101) declared him a paragon of authenticity and spontaneity in poetry, that Tao Yuanming would achieve lasting literary fame.', ' He is also regarded as the foremost representative of what would latter be known as Fields and Gardens poetry, a style of landscape poetry that found inspiration in the beauty and serenity of the natural world close at hand.']], ['Al-Qastallani', [\"Shihab al-Din Abu'l-Abbas Ahmad ibn Muhammad ibn Abu Bakr al-Qastallani al-Qutaybi al-Shafi'i, also known as Al-Qastallani was a Sunni Islamic scholar who specialized in hadith and theology.\", ' He owed his literary fame mainly to his exhaustive commentary on the Sahih al-Bukhari entitled \"Irshad al-Sari fi Sharh al-Bukhari\".']], ['White Horse Tavern (New York City)', [\"The White Horse Tavern, located in New York City's borough of Manhattan at Hudson Street and 11th Street, is known for its 1950s and 1960s Bohemian culture.\", ' It is one of the few major gathering-places for writers and artists from this period in Greenwich Village (specifically the West Village) that remains open.', \" The bar opened in 1880 but was known more as a longshoremen's bar than a literary center until Dylan Thomas and other writers began frequenting it in the early 1950s.\", ' Due to its literary fame, in the past few decades the White Horse has become a popular destination among tourists.']], ['Death in the Afternoon (cocktail)', ['Death in the Afternoon, also called the Hemingway or the Hemingway Champagne, is a cocktail made up of absinthe and Champagne invented by Ernest Hemingway.', ' The cocktail shares a name with Hemingway\\'s book \"Death in the Afternoon\", and the recipe was published in \"So Red the Nose, or Breath in the Afternoon\", 1935 cocktail book with contributions from famous authors.', \" Hemingway's original instructions were:\"]], ['Väinö Linna', ['Väinö Linna (\\xa0\\xa0 ) (20 December 1920 – 21 April 1992) was a Finnish author.', ' He gained literary fame with his third novel, \"Tuntematon sotilas\" (\"The Unknown Soldier\", published in 1954), and consolidated his position with the trilogy \"Täällä Pohjantähden alla\" (\"Under the North Star\", published in 1959–1963 and translated into English by Richard Impola).']], ['Mary Poppins (song)', ['\"Mary Poppins\" is a song from the 2015 stage musical \"Love Birds\" with music and lyrics by Robert J. Sherman.', ' It is sung by \"The Original Quack Pack\", a penguin barbershop quartet who resemble the penguins from the 1964 Walt Disney motion picture, \"Mary Poppins\".', ' In dialogue leading up to the song, the penguins explain that while they did know the same nanny, (Mary Poppins) they are not the same penguins as in the movie.', ' The song expresses their longing for the magical nanny of literary fame.']], ['Robert Bridges', [\"Robert Seymour Bridges, OM (23 October 1844 – 21 April 1930) was Britain's poet laureate from 1913 to 1930.\", ' A doctor by training, he achieved literary fame only late in life.', ' His poems reflect a deep Christian faith, and he is the author of many well-known hymns.', ' It was through Bridges’ efforts that Gerard Manley Hopkins achieved posthumous fame.']], ['Hennchata', ['The Hennchata is a cocktail consisting of Hennessy cognac and Mexican rice horchata agua fresca.', ' It was invented by a restaurant owner in San Jose, California.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.561\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7bea3d5542996dd594b847', 'answer': 'Sean Maguire', 'question': 'What actor replaced the Welch actor born in 1978 who played Robin Hood in the first season of \"Once upon a Time\"?', 'supporting_facts': [['Robin Hood (Once Upon a Time)', 0], ['Robin Hood (Once Upon a Time)', 1], ['Robin Hood (Once Upon a Time)', 2], ['Tom Ellis (actor)', 0]], 'context': [['Gary Bleasdale', ['Gary Bleasdale is an English actor born in Liverpool, Lancashire in 1962.', ' Bleasdale has appeared in many television programmes since 1978 when his first role was playing the lead in an episode of the final series of \"Z-Cars\".', ' He played Kevin Dean in \"The Black Stuff\" (1978), and its sequel \"Boys From the Black Stuff\", (1982).', ' He was a regular on \"The Harry Enfield Show\" for ten years playing one of \"The Scousers\".', ' He has also appeared in \"Casualty\", \"Roger Roger\", \"The Bill\" and many other UK television dramas.', ' He played the Sheriff\\'s sergeant in the 2006 BBC adaptation of \"Robin Hood\".', \" Bleasdale played a brute in 'On The Ledge', at The Royal Court Liverpool in April/May 2008 and Terry in 'Lost Soul' at The Royal Court in September 2008.\", ' He also had a part as a bar patron in the \"Ouroboros\" episode of the BBC TV series Red Dwarf.']], ['Arthur Max', ['Arthur Max (born May 1, 1946) is an American production designer.', ' He has been nominated for three Academy Awards: once for his Production Design work on \"Gladiator\" (2000), \"American Gangster\" (2007), and \"The Martian\" (2015).', ' In addition to his Oscar nominations, Max won several other honors for his production design on the film, including the BAFTA, the National Board of Review prize and the Broadcast Film Critics honor.', ' He also collected two \"Excellence in Production Design\" Award2 from the Art Directors Guild, the first for Gladiator and the second for The Martian.', ' He was also nominated for \"Black Hawk Down\", \"Robin Hood\", \"American Gangster\", \"Prometheus\" and \"Panic Room\".', ' After \"[[The Martian (film), Max worked on [[All The Money In The World]] (2017) marking Max\\'s twelfth project for filmmaker Scott, a list of achievements which includes \"[[Exodus: Gods and Kings]]\", \"[[The Counselor]]\", \"[[Kingdom of Heaven (film)|Kingdom of Heaven]]\", \"Robin Hood\" and the aforementioned \"Black Hawk Down\" and \"[[Body of Lies (film)|Body of Lies]]\".', ' He designed Fincher\\'s 1995 thriller, \"[[Seven (1995 film)|Seven\"]]\".']], ['List of Alyas Robin Hood episodes', ['Alyas Robin Hood is a Philippine drama-action series broadcast by GMA Network starring Dingdong Dantes, Megan Young and Andrea Torres.', ' It premiered on September 19, 2016 on GMA Telebabad prime time block and also aired worldwide on GMA Pinoy TV.', ' The first season ended its 23-week run on February 24, 2017, with a total of 115 episodes, and replaced by \"Destined to be Yours\".']], ['Tom Ellis (actor)', ['Tom Ellis (born 17 November 1978) is a Welsh actor, known for playing Dr. Oliver Cousins in the BBC One soap opera \"EastEnders\" and as Gary Preston in \"Miranda.\"', ' He is known to audiences in the United States for two American television shows in which he plays the title character, USA Network\\'s \"Rush\" (as William Tell Rush) and Fox\\'s \"Lucifer\" (as Lucifer).']], ['Robin Hood (Once Upon a Time)', ['Robin of Locksley, later known as Robin Hood, is a fictional character in ABC\\'s television series \"Once Upon a Time\".', ' He is portrayed by British actor/singer Sean Maguire, who became a series regular in the fifth season after making recurring appearances in the third and fourth season.', ' He is the second actor to play the role in the series, as it was first played by Tom Ellis in the second season, but scheduling conflicts prevented Ellis from reprising the role, resulting in Maguire taking the role afterwards.']], ['List of Alyas Robin Hood characters', ['Alyas Robin Hood ( \\u2009\"Alias Robin Hood\" / English title: \"Bow of Justice\") is an ongoing Philippine drama-action series broadcast by GMA Network starring Dingdong Dantes, Megan Young, Andrea Torres and Solenn Heussaff.', ' It premiered on September 19, 2016 on GMA Telebabad primetime block and also aired worldwide on GMA Pinoy TV.', ' The first season ended its 23-week run on February 24, 2017, with a total of 115 episodes, and replaced by \"Destined to be Yours\".', ' A second season is set to premiere on August 14, 2017 replacing My Love from the Star.']], ['Alyas Robin Hood', ['Alyas Robin Hood ( \\u2009\"Alias Robin Hood\" / English title: \"Bow of Justice\") is a Philippine television drama-action series broadcast by GMA Network starring Dingdong Dantes.', ' It premiered on September 19, 2016 on GMA Telebabad primetime block and also aired worldwide on GMA Pinoy TV.', ' The first season ended its 23-week run on February 24, 2017, with a total of 115 episodes, and replaced by \"Destined to be Yours\".', ' The second season premiered on August 14, 2017, replacing \"My Love from the Star\" and occupying the timeslot of \"Mulawin vs. Ravena\".']], ['Broadway Rose Theatre Company', ['In 1991, seven years after meeting in a summer stock production of \"Joseph and the Amazing Technicolor Dreamcoat\" in Shamokin Dam, Pennsylvania, Dan Murphy and Sharon Maroney (married), along with fellow performer Matthew Ryan (a native of Tigard, OR) and his partner Joseph Morkys, decided to move from New York City to start a summer stock theatre in Tigard.', ' The team pooled their savings of $21,000 and in November 1991, Broadway Rose Theatre Company was incorporated as a 501(c)(3).', ' In the summer of 1992, the first season of Broadway Rose Theatre Company was performed at the Deb Fennell Auditorium at Tigard High School.', \" The company produced five mainstage shows and a children's show in eight weeks, with an average audience of 32 people per performance.\", ' The company lost $8,700 in its first season, but the following year the fledgling company received a $3,000 grant from the Metropolitan Arts Commission (a forerunner of the Regional Arts & Culture Council), to help bring the organization out of debt.', ' In 1993, the company produced \"Oklahoma!', '\" with no funds—putting the entire payroll on Dan\\'s personal credit card.', ' The situation resolved itself as audiences picked up.', ' In 1994, the company received a $4,000 grant from the Metropolitan Arts Commission.', ' Audiences averaged 132 people per performance that year—a 313 percent increase from 1992.', \" In 1995, The Collins Foundation provided the company a $5,000 grant, allowing Sharon Maroney to become the company's first paid employee.\", \" Platt Electric Supply became Broadway Rose's first title sponsor in 1996, providing a new level of stable funding (they would stay on as a title sponsor through 2012 when Harvey Platt sold the company).\", ' In 1997, co-founders Matthew Ryan and Joe Morkys left Broadway Rose and returned to New York.', ' That year Broadway Rose held its first drama camp for young performers aged 8–11.', ' Also in 1997, Shoshana Bean, who would later become famous for portraying Elphaba on Broadway in the musical \"Wicked\"\",\" starred in the Broadway Rose production of \"Bye Bye Birdie.\"', \" The company's offices moved from Dan and Sharon's home to a Platt Electric Supply branch office in 1999.\", ' Later that year, the Sherwood Arts Council contracted Broadway Rose to produce \"Broadway Goes Hollywood\", a fundraiser for SAC held at the historic Robin Hood Theater in Sherwood, OR.', \" Broadway Rose's annual budget rose to around $175,000 with ticket sales accounting for just under half of the total, and Dan's general manager position officially became funded, making him an employee rather than a volunteer.\"]], ['Destined to be Yours', ['Destined to be Yours is a 2017 Philippine romantic-comedy and drama television series broadcast by GMA Network and created by GMA Entertainment TV.', ' It premiered on February 27, 2017 replacing the first season of \"Alyas Robin Hood\" on the GMA Telebabad block and worldwide via GMA Pinoy TV.', ' The series is directed by Irene Villamor and headlined by the AlDub love team of Alden Richards and Maine Mendoza.', ' It is their first prime time television series and follows the story of star-crossed lovers Sinag (Mendoza) and Benjie (Richards).', ' The series ended its 13-week run on May 26, 2017 with a total of 63 episodes and is replaced by \"My Love from the Star\".']], ['Alan Wheatley', ['Alan Wheatley (19 April 1907 – 30 August 1991) was an English actor and former radio announcer.', ' He is perhaps best known for playing the polished villain the Sheriff of Nottingham in the 1950s TV series \"The Adventures of Robin Hood\", with Richard Greene playing Robin Hood.', ' In 1951, Wheatley had played Sherlock Holmes in the first TV series about the fictional detective, but no recordings of it are known to exist.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.562\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac3b04f55429939154138b7', 'answer': 'East Lancashire Railway', 'question': 'Which railway is a part of Summerseat and also runs between Heywood and Rawtenstall?', 'supporting_facts': [['Summerseat railway station', 0], ['Summerseat railway station', 1], ['East Lancashire Railway', 0]], 'context': [['Helsinki–Riihimäki railway', ['Helsinki–Riihimäki railway is a railway running between the Helsinki Central railway station and the Riihimäki railway station in Finland.', \" It was opened in 1862 as a part of the Finland's first railway between Helsinki and Hämeenlinna.\", ' The Helsinki commuter rail system also runs by the Helsinki–Riihimäki railway.']], ['Heywood railway station, Victoria', ['Heywood railway station is a disused station on the Portland railway line in the town of Heywood, in the state of Victoria, Australia.', ' The last passenger train between Ararat and Portland was on 12 September 1981, operated by a DRC railcar.', ' The platform and station building are still in place at Heywood, although in a disused condition.', ' Some of the former yard remains as unconnected broad gauge track, with power connections also provided to a work camp area.', ' The former goods shed was removed by October 1983, and the through line was converted to standard gauge in 1995.']], ['The Wise Woman of Hoxton', ['The Wise Woman of Hoxton is a city comedy by the early modern English playwright Thomas Heywood.', ' It was published under the title \"The Wise-Woman of Hogsdon\" in 1638, though it was probably first performed c. 1604 by the Queen\\'s Men company (of which Heywood was a shareholder), either at The Curtain or perhaps The Red Bull.', ' The play is set in Hoxton, an area that at the time was outside the boundaries of the city of London and notorious for its entertainments and recreations.', ' The Victorian critic F. G. Fleay suggested that Heywood, who was also an actor, originally played the part of Sencer.', ' It has often been compared with Ben Jonson\\'s comic masterpiece \"The Alchemist\" (1610)—the poet T. S. Eliot, for example, argued that with this play Heywood \"succeeds with something not too far below Jonson to be comparable to that master\\'s work\".']], ['Swanage railway station', ['Swanage railway station is a railway station located in Swanage, on the Isle of Purbeck in the English county of Dorset.', ' Originally the terminus of a London and South Western Railway (L&SWR) branch line from Wareham, the line and station were closed by British Rail in 1972.', ' It has since reopened as a station on the Swanage Railway, a heritage railway that currently runs from Norden station just north of Corfe Castle to Swanage station.', ' It now also runs to Wareham on certain services, but not on regular services due to signalling problems.']], ['Summerseat railway station', ['Summerseat railway station is a preserved railway station that serves the village of Summerseat in Greater Manchester, England.', ' It is part of the Heritage East Lancashire Railway (which runs for 12 miles from Heywood-Rawtenstall).']], ['East Lancashire Railway', ['The East Lancashire Railway is a 12+1/2 mi heritage railway line in north west England which runs between Heywood and Rawtenstall with intermediate stations at Bury Bolton Street, Burrs Country Park , Summerseat, Ramsbottom and Irwell Vale.']], ['Rawtenstall to Bacup Line', ['The Rawtenstall to Bacup railway line opened in two stages, from Rawtenstall to Waterfoot in 1848, and from Waterfoot to the Bacup terminus in 1852.', ' There were stations at Rawtenstall, Cloughfold, Stacksteads and Bacup.', ' The line was doubled in 1880, at the same time as the line from Bacup to Rochdale was also opened (closed 1947).', ' Passenger and freight services operated until the Beeching cuts in 1966, the last passenger train running on December 5th 1966 and the track being lifted in 1969.']], ['South Coast Line', ['The South Coast Line is an intercity rail service operated by NSW TrainLink that services the Illawarra region of New South Wales, Australia.', ' The service runs from , and runs the entire length of the eponymous South Coast railway line to .', ' The service also runs along the Eastern Suburbs railway line at peak hours and the Port Kembla railway line to .', ' It is operated with NSW TrainLink H sets and Sydney Trains T sets, with Endeavour railcars operating the service on the non-electrified line between and Bomaderry.']], ['Don River Railway', ['The Don River Railway is a volunteer run vintage railway and museum in Don, a suburb of Devonport, Tasmania.', ' It runs a passenger train ride from Don to Coles Beach and return using part of the former Don River Line that ran between Don Junction and Paloona.', ' The service is usually run Wednesday through to Sunday inclusive.', ' Don River Railway is open every day except Good Friday and Christmas Day.', ' On operating weekdays, customer can expect to ride in either a 1940s ex TGR rail car, or diesel locomotive pulling heritage carriages, whilst steam locomotives usually run on weekends.', ' .', ' The railway also runs a number of diesel locomotives numbered V2, X4, Y6 and 866.', ' Numerous other locomotives and rolling stock are in the process of being restored at the on-site workshop.']], ['Southern Railway of British Columbia', ['The Southern Railway of British Columbia, branded as SRY Rail Link (reporting mark SRY) is a Canadian short line railway operating in the southwestern British Columbia.', ' The main facility is the port at Annacis Island with major import of cars, export of forestry products, and other shipments.', ' The railway has interconnections with three Class I railroads, including Canadian Pacific (CP), Canadian National (CN) and Burlington Northern Santa Fe (BNSF).', ' It operates a fleet of 29 locomotives, mostly consisting of EMD GP-9 & SW900 locomotives.', ' It also rosters 5 unique Ex.', ' Canadian National Railway GMD-1 locomotives, and also runs 3 SD38-2 locomotives, and 1 SD38AC.', ' The railroad also operates a fleet 2,000 rail cars, SRY hauls approximately 70,000 carloads per year.', ' It operates around 123 mi of track, 62 mi of which is mainline track.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.562\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae6196d5542995703ce8afd', 'answer': 'The Mariinsky Ballet', 'question': 'Arthur Saint-Léon was the \"Maître de Ballet\" of the ballet company now known as?', 'supporting_facts': [['Arthur Saint-Léon', 0], ['Mariinsky Ballet', 0], ['Mariinsky Ballet', 1], ['Mariinsky Ballet', 3]], 'context': [['Mariinsky Ballet', ['The Mariinsky Ballet is the resident classical ballet company of the Mariinsky Theatre in Saint Petersburg, Russia.', \" Founded in the 18th century and originally known as the Imperial Russian Ballet, the Mariinsky Ballet is one of the world's leading ballet companies.\", ' Internationally, the Mariinsky Ballet continues to be known by its former Soviet name the Kirov Ballet.', ' The Mariinsky Ballet is the parent company of the \"Vaganova Ballet Academy\", a leading international ballet school.']], ['Listing of the works of the Maître de Laz', ['This is a listing/\"catalogue raisonnė\" of the works of the Maître de Laz.', ' His work, dating to around 1527, can be seen in various parts of Brittany.', ' He is unusual amongst sculptors in this region working in the 15th and 16th century in that he often worked using \"grés feldspathique\" as opposed to granite or kersanton stone.', \" He executed pietàs in Laz, Finistère, Briec-de-l'Odet, Saint-Hernin and Plourac'h and in Plourac'h, he executed statuary for the parish church.\", ' Grés feldspathique had been used earlier in Laz in 1350 with a statue of a dying cavalier placed by the chevet of the Église Saint-Germain-et-Saint-Louis.', ' It was also used by the Maître de Tronoën (Listing of the works of the atelier of the Maître de Tronoën.', ') for the bas-relief in the entry to the old presbytery at Laz.']], ['Jean-Louis Aumer', ['Jean-Louis Aumer was a French danseur and choreographer, who was born in Strasbourg on 21 April 1774, and who died in Saint-Martin-de-Boscherville in July 1833.', ' Educated at the school of the Paris Opera Ballet, he joined the company in 1801 after an initial engagement with Jean Dauberval in Bordeaux.', ' The Paris Opera\\'s \"maître de ballet\" Pierre Gardel presented an obstacle which led Aumer to choose the Théâtre de la Porte Saint-Martin as the venue for which to create his early ballets.', ' Faced with the implacable competition from Gardel, Aumer left France for engagements in Kassel (1808–1814) and Vienna (1814–1820).', ' Brief periods in Paris (1821–1822) and London (1824–1825) were followed by his return to the Paris Opera Ballet (1820–1831), where, enriched by the experience of working abroad, he engaged in a profound renovation of the French repertory, capped by his \"chef-d\\'œuvre\", \"Manon Lescaut\" (1830).']], ['Pierre Tabart', ['Pierre Tabart (also Thabart, Tharbart) (Chinon, baptized 8 January 1645 – Meaux, 1716) was a French composer and maître de chapelle.', \" Said to have studied music under 'the best contrapuntist of his time', he served as maître de musique in Orléans until 1683, followed by Senlis from 1683-1689.\", ' He then succeeded Nicolas Goupillet as maître de musique of Meaux Cathedral.', ' However, due to the fact that his nine-year contract was left unrenewed, he was succeeded by Sébastien de Brossard in 1699.', ' He later aided his successor to the post in selecting the maître de musique for Évreux Cathedral.']], ['Sergeyev Collection', ['The Sergeyev Collection is a collection of choreographic notation, music, designs for décor and costumes, theatre programs, photos and other materials that document the repertory of the Imperial Ballet (precursor of the Kirov/Mariinsky Ballet) of St. Petersburg, Russia at the turn of the 20th century.', ' The majority of the choreographic notations document with varying degrees of detail the original works and revivals of the renowned choreographer Marius Petipa, who served as \"Premier Maître de ballet\" of the St. Petersburg Imperial Theatres, as well as notation and music documenting the ballets of Lev Ivanov, who served as second \"Maître de ballet\".', ' Also included in the collection are choreographic notation documenting dances from various operas by both Petipa and Ivanov, respectively.']], ['Cesare Pugni', ['Cesare Pugni (Russian: Цезарь Пуни ) (] ; 31 May 1802–26 January\\xa0[O.S. 14 January]\\xa01870 ) born in Genoa, was an Italian composer of ballet music, a pianist and a violinist.', ' In his early career he composed operas, symphonies, and various other forms of orchestral music.', \" Pugni is most noted for the ballets he composed for Her Majesty's Theatre in London (1843–1850), and for the Imperial Theatres in St. Petersburg, Russia (1850–1870).\", ' The majority of his ballet music was composed for the works of the ballet master Jules Perrot, who mounted nearly every one of his ballets to scores by Pugni.', ' In 1850 Perrot departed London for Russia, having accepted the position of \"Premier maître de ballet\" of the St. Petersburg Imperial Theatres at the behest of Carlotta Grisi, who was engaged as \"Prima ballerina\".', \" Cesare Pugni followed Perrot and Grisi to Russia, and remained in the imperial capital even after Grisi's departure in 1853 and Perrot's departure in 1858.\", \" Pugni went on the compose for Perrot's successors Arthur Saint-Léon and Marius Petipa, serving as the Imperial Theatre's official composer of ballet music until his death in 1870.\"]], ['Ballet master', ['Ballet Master (also \"Balletmaster\", \"Ballet Mistress\" [increasingly archaic English language use], \"Premier Maître de ballet\" or \"Premier Maître de ballet en Chef\") is the term used for an employee of a ballet company who is responsible for the level of competence of the dancers in their company.', \" In modern times, ballet masters are generally charged with teaching the daily company ballet class and rehearsing the dancers for both new and established ballets in the company's repertoire.\", ' The artistic director of a ballet company, whether a male or female, may also be called its ballet master.', ' Historic use of gender marking in job titles in ballet (and live theatre) is being supplanted by gender-neutral language job titles regardless of an employee\\'s gender identity or expression (e.g. \"Ballet Master\" in lieu of \"Ballet Mistress\", \"Wig Master\" as an alternative to \"Wig Mistress\").']], ['Louis Mérante', ['Louis Alexandre Mérante (23 July 1828–Courbevoie, 17 July 1887) was a dancer and choreographer, the \"Maître de Ballet\" (First Balletmaster/Chief Choreographer) of the Paris Opera Ballet at the Salle Le Peletier until its destruction by fire in 1873, and subsequently the first Ballet Master at the company\\'s new Palais Garnier, which opened in 1875.', ' He is best remembered as the choreographer of Léo Delibes\\' \"Sylvia, ou la nymphe de Diane\" (1876).', ' With Arthur Saint-Léon and Jules Perrot, he is one of the three choreographers who defined the French ballet tradition during the Second French Empire and the Third Republic according to choreographer .']], ['Agrippina Vaganova', ['Agrippina Yakovlevna Vaganova (Russian: Агриппина Яковлевна Ваганова ; 26 June 1879 – 5 November 1951) was a Russian ballet teacher who developed the Vaganova method – the technique which derived from the teaching methods of the old \"Imperial Ballet School\" (today the \"Vaganova Academy of Russian Ballet\") under the \"Premier Maître de Ballet\" Marius Petipa throughout the mid to late 19th century, though mostly throughout the 1880s and 1890s.', ' It was Vaganova who perfected and cultivated this form of teaching the art of classical ballet into a workable syllabus.', ' Her \"Fundamentals of the Classical Dance\" (1934) remains a standard textbook for the instruction of ballet technique.', ' Her technique is one of the most popular techniques today.']], ['Arthur Saint-Léon', ['Arthur Saint-Léon (17 September 1821, Paris – 2 September 1870) was the \"Maître de Ballet\" of St. Petersburg Imperial Ballet from 1859 until 1869 and is famous for creating the choreography of the ballet \"Coppélia\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.563\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8ec55f5542995a26add50c', 'answer': '4145 ft', 'question': 'At what height does State Route 160 start?', 'supporting_facts': [['Virginia State Route 160', 0], ['Virginia State Route 160', 1], ['Black Mountain (Kentucky)', 0]], 'context': [['Virginia State Route 160', ['State Route 160 (SR 160) is a primary state highway in the U.S. state of Virginia.', ' Known as the Trail of the Lonesome Pine, the state highway runs 8.02 mi from the Kentucky state line on top of Black Mountain, where the highway continues north as Kentucky Route 160 (KY 160), east to SR 68 in Appalachia.']], ['Alabama State Route 160', ['State Route\\xa0160 (SR\\xa0160) is an 18.415 mi state highway in Blount County, in the north-central part of the U.S. state of Alabama.', ' The western terminus of the highway is at an intersection with U.S. Route\\xa031 (US\\xa031) on the southern edge of Smoke Rise.', \" This intersection is just east of US\\xa031's interchange with Interstate\\xa065 (I-65 exit 284).\", ' The eastern terminus of the highway is at an intersection with US\\xa0231 and Blount County Route\\xa01 (CR\\xa01) in Cleveland.', ' SR\\xa0160 is the only state highway that is exclusively located in Blount County.']], ['Washington State Route 160', ['State Route\\xa0160 (SR\\xa0160) is a 7.47 mi long state highway serving Kitsap and King counties in the U.S. state of Washington.', ' The highway begins at an interchange with SR\\xa016 in Port Orchard and travels east to the Southworth ferry terminal, where the route continues onto a ferry to Vashon Heights, the former southern terminus of SR\\xa0339, and further east to end at the Fauntleroy ferry terminal in Seattle.']], ['Maine State Route 160', [\"State Route 160 (abbreviated SR\\xa0160) is part of Maine's system of numbered state highways, located in the western part of the state.\", ' It is a north–south highway running 32.64 mi from an intersection with State Route 5 in Limerick to an intersection with State Route 117 in Denmark.']], ['California State Route 104', [\"State Route 104 (SR 104) is a west–east state highway in California's Central Valley.\", ' It connects State Route 99 near Galt to State Route 49 in Sutter Creek via the city of Ione.', ' It is known as Twin Cities Road from its western terminus up until just before Ione.', ' Heading west past its western terminus along Twin Cities Road will lead to Interstate 5 and eventually to an end at State Route 160 north of Walnut Grove.']], ['Arizona State Route 564', ['State Route 564, also known as SR 564, is a state highway in northern Arizona serving Navajo National Monument.', ' This highway travels from U.S. Route 160 to Betatakin Ruin; SR 564 derives its number from the former route number of the adjacent stretch of US 160, U.S. Route 164.', ' SR 564 ends at Betatakin; smaller roads travel beyond to Keet Seel.']], ['Ohio State Route 160', ['State Route 160 (SR 160) is a north–south state highway in the southeastern portion of the U.S. state of Ohio.', ' Its southern terminus is at SR 7 in Gallipolis, and the route heads north.', ' It meets U.S. Route 35 at an interchange with various collector and distributor ramps.', ' SR 160 southbound bypasses the interchange on a 0.739 mi road officially designated SR 160-A.', ' From there, the route passes through Vinton in northern Gallia County.', ' Following an intersection with SR 32, the route heads in a more westerly direction until it meets and its northern terminus is at State Route 93 in Hamden.']], ['California State Route 220', ['State Route 220 (SR 220) is a state highway in the U.S. state of California, defined to run between State Route 84 and State Route 160 on Ryer Island.', ' At the eastern end of Ryer Island, the road crosses Steamboat Slough on the Howard Landing Ferry, a cable ferry.']], ['Kentucky Route 160', ['Kentucky Route 160, also known as KY 160, is a state highway in the U.S. state of Kentucky.', ' It runs from the Virginia state line, where the roadway continues east to Appalachia, Virginia as State Route 160, north via Lynch, Benham, Clutts, Cumberland, Sand Hill, Gordon, Linefork, Kings Creek, Premium, and Hot Spot to Kentucky Route 15 at Van.', ' KY 160 overlaps KY 15 through Isom to Cody, where it splits to run via Carr Creek, Brinkley, and Hindman, ending at Kentucky Route 1087 at Vest.']], ['Black Mountain (Kentucky)', ['Black Mountain is the highest mountain peak in the Commonwealth of Kentucky, USA, with a summit elevation of 4145 ft above mean sea level and a top to bottom height of over 2500 ft .', ' The summit is located at approximately in Harlan County, Kentucky near the Virginia border, just above the towns of Lynch, Kentucky and Appalachia, Virginia.', ' It is about 500 ft taller than any other mountain in Kentucky.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.565\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae143ed55429920d5234360', 'answer': '1755', 'question': 'In what year was the university where Sergei Aleksandrovich Tokarev was a professor founded?', 'supporting_facts': [['Sergei Aleksandrovich Tokarev', 0], ['Moscow State University', 3]], 'context': [['Moscow State University', ['Lomonosov Moscow State University (MSU; Russian: Московский государственный университет имени М.', ' В.', ' Ломоносова , often abbreviated МГУ) is a coeducational and public research university located in Moscow, Russia.', ' It was founded on January 25, 1755 by Mikhail Lomonosov.', ' MSU was renamed after Lomonosov in 1940 and was then known as \"Lomonosov University\".', ' It also houses the tallest educational building in the world.', ' It is rated among the universities with the best reputation in the world.', ' Its current rector is Viktor Sadovnichiy.']], ['Sergei Roshchin', ['Sergei Aleksandrovich Roshchin (Russian: Серге́й Александрович Рощин ; born January 28, 1989) is a Russian football defender, who last played for FC Znamya Truda Orekhovo-Zuyevo.']], ['Sergei Kornilenko', ['Sergei Aleksandrovich Kornilenko (Belarusian: Сяргей Аляксандравіч Карніленка ; Russian: Сергей Александрович Корниленко; born 14 June 1983) is a Belarusian professional footballer who plays as a striker for FC Krylia Sovetov Samara of the Russian Premier League.', ' In Belarus, both Belarusian and Russian languages are official.', ' Thus his name, usually transliterated as Sergei Kornilenko (Russian: Серге́й Корниленко ), can be alternatively spelled as Syarhey Karnilenka (Belarusian: Сяргей Карніленка ).']], ['Sergei Chikildin', ['Sergei Aleksandrovich Chikildin (Russian: Серге́й Александрович Чикильдин ; born January 25, 1991) is a Russian football goalkeeper, who last played for FC Kavkaztransgaz-2005 Ryzdvyany.']], ['Sergei Sholokhov', ['Sergei Aleksandrovich Sholokhov (Russian: Серге́й Александрович Шолохов ; born September 6, 1980) is a Russian professional football player.', ' As of August 2009, he plays in the Russian Second Division for FC Avangard Kursk.', ' Before 2004 he was known as Sergei Kocherga (Russian: Серге́й Кочерга ).']], ['Sergei Panchin', ['Sergei Aleksandrovich Panchin (Russian: Серге́й Александрович Панчин ; born 15 December 1993) is a Russian football goalkeeper.']], ['Sergei Kosarev', ['Sergei Aleksandrovich Kosarev (Russian: Серге́й Александрович Косарев ; born January 29, 1993) is a Russian football midfielder, who currently plays for FC MITOS Novocherkassk.']], ['Sergei Dmitrochenko', ['Sergei Aleksandrovich Dmitrochenko (Russian: Серге́й Александрович Дмитроченко ; born June 21, 1993) is a Russian football midfielder.']], ['Sergei Aleksandrovich Tokarev', ['Sergei Aleksandrovich Tokarev (Russian: Серге́й Алекса́ндрович То́карев , 29 December 1899 – 19 April 1985) was a Russian scholar, ethnographer, historian, researcher of religious beliefs, doctor of historical sciences, and professor at Moscow State University.']], ['Sergei Aleksandrovich Kudryavtsev', ['Sergei Aleksandrovich Kudryavtsev (1903 – April 25, 1938) was a Ukrainian communist Soviet politician.', ' He was born in Kharkiv.', ' During the Great Purge, he was arrested on October 13, 1937 and later executed by firing squad.', ' After the death of Joseph Stalin, he was rehabilitated in 1956.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.565\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae3fe635542996836b02c07', 'answer': \"Martin O'Malley\", 'question': 'Which American politician was the 61st Governor of Maryland and had help on his first campaign in 1998 from Richard Burton?', 'supporting_facts': [['Richard Burton (Baltimore)', 1], [\"Martin O'Malley\", 0]], 'context': [[\"Martin O'Malley presidential campaign, 2016\", [\"The 2016 presidential campaign of Martin O'Malley, the 61st Governor of Maryland, for the Democratic nomination for President of the United States in 2016 was announced on May 30, 2015.\", ' On February 1, 2016, he suspended his campaign after a poor showing in the Iowa caucuses.']], [\"Martin O'Malley\", [\"Martin Joseph O'Malley (born January 18, 1963) is an American politician and attorney who served as the 61st Governor of Maryland from 2007 to 2015.\", ' He previously served as the Mayor of Baltimore from 1999 to 2007, and was a councilman from the Third Councilmanic District in the northeast section of the city on the Baltimore City Council from 1991 to 1999.']], ['61st Cavalry Division (Soviet Union)', ['The 61st Cavalry Division was a cavalry division of the Red Army that served in the first years of the Great Patriotic War.', ' It was formed in September – October, 1941, and saw its first actions to the south of Stalingrad during the German siege of that city in the autumn of 1942.', ' When the Soviet counteroffensive, Operation Uranus, began in November the 61st formed a significant part of the mobile forces of its 51st Army.', ' After the positions of Romanian 4th Army were broken through the division took part in the exploitation to the southwest, but became overextended and vulnerable to the mobile German reinforcements arriving to attempt a breakthrough to their Sixth Army.', ' The 61st suffered such severe losses that it had to be withdrawn to the reserves in December, and was later disbanded.']], ['Kirk Fordice', ['Daniel Kirkwood \"Kirk\" Fordice Jr. (February 10, 1934 – September 7, 2004), was an American politician and businessman who served as the 61st Governor of Mississippi from January 14, 1992 until January 11, 2000.', ' He was the first Republican governor of the state since Reconstruction-era governor Adelbert Ames, who served from 1874 to 1876.']], ['Henry Roberts (governor)', ['Henry Roberts (January 22, 1853 – May 1, 1929) was an American politician who was the 61st Governor of Connecticut.']], ['Richard Burton (Baltimore)', ['Richard Burton is a Baltimore, Maryland, city council employee and runs the \"Believe\" campaign.', \" He was a rapper before becoming involved in Martin O'Malley's first Mayoral campaign after meeting him in 1998.\"]], ['Military Transport Aviation', ['Military Transport Aviation Command (Russian: Кома́ндование вое́нно-тра́нспортной авиа́ции (ВТА) — \" Komandovaniye voyenno-transportnoy aviatsii (VTA)\") was a major component of the former Soviet Air Forces, active from the Cold War period, through the dissolution of the Soviet Union, to 1998–1999.', ' In 1999–2009 it was reduced in status to the 61st Air Army of the Supreme High Command (\"61 Vozdushnaya Armiya VGK\").', ' The 61st Air Army itself was initially formed on 10 January 1949 by renaming the 3rd Air Army.', ' In 2009 the 61st Air Army was renamed the Command of']], ['John E. Weeks', ['John Eliakim Weeks (June 14, 1853 – September 10, 1949) was an American politician from Vermont.', ' He served as the 61st Governor of Vermont from 1927 to 1931.']], ['Linwood Holton', ['Abner Linwood Holton Jr. (born September 21, 1923) is a Virginia political figure and attorney.', ' He served as the 61st Governor of Virginia, from 1970 to 1974.', ' He was the first Republican governor of Virginia in the 20th Century.', ' He was also the first Republican who won a popular election as governor.', ' Holton is the father of Anne, and the father-in-law of Tim Kaine.']], ['Steve Beshear', ['Steven Lynn \"Steve\" Beshear (born September 21, 1944) is an American attorney and politician who served as the 61st governor of Kentucky from 2007 to 2015.', \" He served in the Kentucky House of Representatives from 1974 to 1980, was the state's Attorney General from 1980 to 1983, and was the 49th lieutenant governor from 1983 to 1987.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.566\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab9ddaa5542994dbf019873', 'answer': 'Lord Gort', 'question': 'Brigadier Stanley James Ledger Hill was attached to the command post of which senior British Army officer born on July 10, 1886?', 'supporting_facts': [['James Hill (British Army officer)', 2], ['John Vereker, 6th Viscount Gort', 0]], 'context': [['James Hill (British Army officer)', ['Brigadier Stanley James Ledger Hill {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} & Two Bars, MC (14 March 1911 – 16 March 2006) was a British Army officer, who served as commander of the 3rd Parachute Brigade, part of the 6th Airborne Division, during World War II.', ' Born in Bath, Somerset, Hill was educated at Marlborough College and the Royal Military College, Sandhurst before joining the British Army in 1931 and being commissioned into the Royal Fusiliers.', ' He commanded a platoon for a short period, and was then attached to the command post of Field Marshal Lord Gort during the Battle of France in May 1940, where he oversaw the evacuation of Brussels as well as the beach at De Panne during the evacuation of Dunkirk.', ' After a brief period of time in the Irish Free State, he volunteered for parachute training and joined the 1st Parachute Battalion, and was its commanding officer when its parent formation, the 1st Parachute Brigade, was deployed to North Africa.']], [\"Richard O'Connor\", [\"General Sir Richard Nugent O'Connor & Bar, MC (21 August 1889 – 17 June 1981) was a senior British Army officer who fought in both the First and Second World Wars, and commanded the Western Desert Force in the early years of the Second World War.\", ' He was the field commander for Operation \"Compass\", in which his forces destroyed a much larger Italian army – a victory which nearly drove the Axis from Africa, and in turn, led Adolf Hitler to send the German Africa Corps under Erwin Rommel to try to reverse the situation.', \" O'Connor was captured by a German reconnaissance patrol during the night of 7 April 1941 and spent over two years in an Italian prisoner of war camp.\", ' He eventually escaped after the fall of Mussolini in the autumn of 1943.', ' In 1944 he commanded VIII Corps in the Battle of Normandy and later during Operation Market Garden.', ' In 1945 he was General Officer in Command of the Eastern Command in India and then, in the closing days of British rule in the subcontinent, he headed Northern Command.', \" His final job in the army was Adjutant-General to the Forces in London, in charge of the British Army's administration, personnel and organisation.\"]], ['Herbert Plumer, 1st Viscount Plumer', ['Field Marshal Herbert Charles Onslow Plumer, 1st Viscount Plumer, {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} (13 March 1857 – 16 July 1932) was a senior British Army officer of the First World War.', ' After commanding V Corps at the Second Battle of Ypres in April 1915, he took command of the Second Army in May 1915 and in June 1917 won an overwhelming victory over the German Army at the Battle of Messines, which started with the simultaneous explosion of a series of mines placed by the Royal Engineers\\' tunnelling companies beneath German lines, which created 19 large craters and was described as the \"loudest explosion in human history\".', ' He later served as Commander-in-Chief of the British Army of the Rhine and then as Governor of Malta before becoming High Commissioner of the British Mandate for Palestine in 1925 and retiring in 1928.']], ['Herbert Lumsden', ['Lieutenant-General Herbert Lumsden & Bar, MC (8 April 1897 – 6 January 1945) was a senior British Army officer who fought in both World War I and World War II.', ' He was the most senior British Army combat casualty of the Second World War.']], ['Matthew Maer', [\"Brigadier Matthew Philip Maer DSO, MBE is a senior British Army officer of the Princess of Wales's Royal Regiment.\", \" Maer commanded 1st Battalion, Princess of Wales's Royal Regiment as a Lieutenant-Colonel, deploying to Maysan, Iraq in 2004/5 in the aftermath of the invasion.\", ' In this role, he was awarded the Distinguished Service Order for his leadership, which included command of Victoria Cross recipient Johnson Beharry.', ' Maer was promoted colonel on 30 June 2006, and brigadier on 30 June 2009.', ' He was appointed Member of the Order of the British Empire on 6 November 1998.']], ['John Dill', ['Field Marshal Sir John Greer Dill, (25 December 1881 – 4 November 1944) was a senior British Army officer with service in both the First World War and the Second World War.', ' From May 1940 to December 1941 he was the Chief of the Imperial General Staff (CIGS), the professional head of the British Army, and subsequently in Washington, D.C., as Chief of the British Joint Staff Mission and then Senior British Representative on the Combined Chiefs of Staff (CCS), played a significant role during the Second World War in the formation of the \"Special Relationship\" between the United Kingdom and the United States.']], ['Nick Carter (British Army officer)', ['General Sir Nicholas Patrick \"Nick\" Carter, {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} (born 11 February 1959) is a senior British Army officer.', ' He served as commanding officer of 2nd Battalion, Royal Green Jackets in which role he was deployed to Bosnia in 1998 and Kosovo in 1999.', ' After service in Afghanistan, he took command of 20th Armoured Brigade in 2004 and commanded British forces in Basra.', ' He was subsequently appointed General Officer Commanding 6th Division, which was deployed to Afghanistan with Carter as Commander ISAF Regional Command South, before he became Director-General Land Warfare.', ' After that he became Deputy Commander Land Forces in which role he was the main architect of the Army 2020 concept.', ' After a tour as Deputy Commander, International Security Assistance Force, he assumed the position of Commander Land Forces in November 2013.', ' In September 2014, he became head of the British Army as Chief of the General Staff succeeding General Sir Peter Wall.']], ['John Vereker, 6th Viscount Gort', ['Field Marshal John Standish Surtees Prendergast Vereker, 6th Viscount Gort & Two Bars, (10 July 1886 – 31 March 1946) was a senior British Army officer.', ' As a young officer during the First World War he was decorated with the Victoria Cross for his actions during the Battle of the Canal du Nord.', ' During the 1930s he served as Chief of the Imperial General Staff (the professional head of the British Army).', ' He is most famous for commanding the British Expeditionary Force sent to France in the first year of the Second World War, which was evacuated from Dunkirk.', ' Gort later served as Governor of Gibraltar and Malta, and High Commissioner for Palestine and Transjordan.']], ['Ledger Hill', ['Arthur James Ledger Hill (26 July 1871 in Bassett, Hampshire – 6 September 1950 in Spursholt House, Romsey, Hampshire) was an English cricketer.']], ['Commander Regional Forces (United Kingdom)', ['The Commander Regional Forces (CRF) was a senior British Army officer who had command over the \"Regenerative Divisions\" of the British Army i.e. those divisions that are not on full strength and would only be mobilised in a national emergency.', ' The post was held by a Lieutenant General and was based at HQ Land Forces.', ' The post holder was also Inspector-General of the Territorial Army.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.566\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab849d955429916710eb017', 'answer': '11 November 1918', 'question': 'The Battle of Cambrai took place during a series of offensives that ended on what date?', 'supporting_facts': [['Battle of Cambrai (1918)', 0], ['Hundred Days Offensive', 0]], 'context': [['Somaliland Campaign', ['The Somaliland Campaign, also called the Anglo-Somali War or the Dervish War, was a series of military expeditions that took place between 1900 and 1920 in the Horn of Africa, pitting the Dervishes led by Mohammed Abdullah Hassan (nicknamed the \"Mad Mullah\", although he \"was neither mad nor a mullah\") against the British.', ' The British were assisted in their offensives by the Ethiopians and Italians.', ' During the First World War (1914–1918), Hassan also received aid from the Ottomans, Germans and, for a time, from the Emperor Iyasu V of Ethiopia.', ' The conflict ended when the British aerially bombed the Dervish capital of Taleh in February 1920.']], ['To the Green Fields Beyond (game)', ['To the Green Fields Beyond is a game created in 1978 by SPI, or Simulations Publications Incorporated.', ' It is about the battle of Cambrai, which took place from November 20 to December 7, 1917.', ' At Cambrai, the British and the French tried to use the newly invented tank (land ship as it was called at the time) to break through German front lines.']], ['Hundred Days Offensive', ['The Hundred Days Offensive was the final period of the First World War, during which the Allies launched a series of offensives against the Central Powers on the Western Front from 8 August to 11 November 1918, beginning with the Battle of Amiens.', ' The offensive essentially pushed the Germans out of France, forcing them to retreat beyond the Hindenburg Line, and was followed by an armistice.', ' The term \"Hundred Days Offensive\" does not refer to a specific battle or unified strategy, but rather the rapid series of Allied victories starting with the Battle of Amiens.']], ['Battle for Henderson Field', ['The Battle for Henderson Field, also known as the Battle of Henderson Field or Battle of Lunga Point by the Japanese, took place from 23–26 October 1942 on and around Guadalcanal in the Solomon Islands.', ' The battle was a land, sea, and air battle of the Pacific campaign of World War II and was fought between the Imperial Japanese Army and Navy and Allied (mainly United States (U.S.) Marine and U.S. Army) forces.', ' The battle was the third of the three major land offensives conducted by the Japanese during the Guadalcanal campaign.']], ['Battle of Courtrai (1918)', ['The Battle of Courtrai (also known as the Second Battle of Belgium (French: \"2ème Bataille de Belgique\" ) and the Battle of Roulers (French: \"Bataille de Roulers\" )) was one of a series of offensives in northern France and southern Belgium that took place in late September and October 1918.']], ['Siege of Cambrai (1677)', ['The Siege of Cambrai took place from 20 March to 19 April 1677 during the Franco-Dutch War.']], ['Battle of El Herri', ['The Battle of El Herri (also known as Elhri) was fought between France and the Berber Zaian Confederation on 13 November 1914.', ' It took place at the small settlement of El Herri, near Khénifra in the French protectorate in Morocco.', ' The battle was part of the Zaian War, in which the confederation of tribes sought to oppose continued French expansion into the interior of Morocco.', ' Having captured the strategic town of Khénifra earlier in the year, the French, under General Hubert Lyautey, entered negotiations with Mouha ou Hammou Zayani, who led the Zaian.', ' Lyautey thought that peace could be achieved and ordered Lieutenant-Colonel René Laverdure, who commanded the garrison in Khénifra, not to launch any offensives.']], ['Battle of Cambrai (1918)', ['The Battle of Cambrai, 1918 (also known as the Second Battle of Cambrai) was a battle between troops of the British First, Third and Fourth Armies and German Empire forces during the Hundred Days Offensive of the First World War.', ' The battle took place in and around the French city of Cambrai, between 8 and 10 October 1918.', ' The battle incorporated many of the newer tactics of 1918, in particular tanks.', ' The attack was an overwhelming success with light casualties in an extremely short amount of time.']], ['Battle of La Motta (1513)', ['The Battle of La Motta, also known as the Battle of Schio, Battle of Vicenza or Battle of Creazzo, took place at Schio, in the Italian region of Veneto, Republic of Venice, on 7 October 1513, between the forces of the Republic of Venice and a combined force of Spain and the Holy Roman Empire, and was a significant battle of the War of the League of Cambrai.', \" A Venetian army under Bartolomeo d'Alviano was decisively defeated by the Spanish/Imperial army commanded by Ramón de Cardona and Fernando d'Avalos.\"]], [\"Battle of Edson's Ridge\", [\"The Battle of Edson's Ridge, also known as the Battle of the Bloody Ridge, Battle of Raiders Ridge, and Battle of the Ridge, was a land battle of the Pacific campaign of World War II between Imperial Japanese Army and Allied (mainly United States Marine Corps) ground forces.\", ' It took place from 12–14 September 1942, on Guadalcanal in the Solomon Islands, and was the second of three separate major Japanese ground offensives during the Guadalcanal Campaign.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.568\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a77415b55429966f1a36cd2', 'answer': 'Martin \"Marty\" McCann', 'question': 'What actor from Northern Ireland born in 1983 starred in Whole Lotta Sole?', 'supporting_facts': [['Whole Lotta Sole', 0], ['Martin McCann (actor)', 0]], 'context': [['Martin McCann (actor)', ['Martin \"Marty\" McCann (born 20 July 1983) is an actor from Northern Ireland.']], ['Northern Ireland Association of Aeromodellers', ['The Northern Ireland Association of Aeromodellers (NIAA), is the governing body for all British Model Flying Association (BMFA) affiliated clubs in Northern Ireland.', ' The NIAA committee is elected by, and from representatives of the local Northern Ireland flying clubs, with an aim to promote, protect, organise and encourage model aircraft flying throughout Northern Ireland.', ' Additionally the organisation coordinates local club events and activities, and acts as a contact and representative on their behalf to the BMFA.', ' The NIAA is accredited in these rolls by the Sports Council for Northern Ireland (SCNI), and the BMFA who are delegated by the Royal Aero Club to be responsible for all aspects of model flying in Great Britain.', ' The BMFA is also recognised as the sole representative organisation for the sport in the UK by the Fédération Aéronautique Internationale (FAI) which is the world wide governing body for all forms of sporting aviation, including model flying.']], ['John Cameron (musician)', ['John Cameron (born 20 March 1944, Woodford, Essex, England) is a British composer, arranger, conductor and musician.', ' He is well known for his many film, TV and stage credits, and for his contributions to \"pop\" recordings, notably those by Donovan, Cilla Black and the group Hot Chocolate.', ' Cameron\\'s instrumental version of Led Zeppelin\\'s \"Whole Lotta Love\", became a hit for his group Collective Consciousness Society and, for many years, a version of Cameron\\'s arrangement was used as the theme music for the BBC TV show, \"Top of the Pops\".']], [\"There's a Whole Lot of Loving\", ['\"There\\'s a Whole Lot of Loving\" is a 1975 hit song by Guys \\'n\\' Dolls written by Christian Arnold with lyrics by David Martin and Geoff Morrow.', ' The song was a #2 hit in both the UK and Ireland, and became the biggest hit for the group.', ' The song was covered as \"Whole Lotta Lovin\\'\" by Six and was a number one single in Ireland in 2002.']], ['Great Britain at the 2016 Summer Olympics', ['The United Kingdom of Great Britain and Northern Ireland competed as Great Britain at the 2016 Summer Olympics in Rio de Janeiro, Brazil, from 5 to 21 August 2016.', ' The United Kingdom was represented by the British Olympic Association (BOA), and the team of selected athletes was also known as Team GB.', ' British athletes have appeared in every Summer Olympic Games of the modern era, alongside Australia, France, Greece, and Switzerland, though Great Britain is the only country to have won at least one gold medal at all of them.', ' Although the British Olympic Association is the National Olympic Committee (NOC) for Great Britain and Northern Ireland, Northern Irish athletes can choose whether to compete for Great Britain or for the Republic of Ireland, as they are entitled to citizenship of either nation under the Good Friday Agreement.', \" In 2016 Northern Ireland born representatives in Team GB included returning rowers Alan Campbell, Peter Chambers and Richard Chambers, archer Patrick Huston and four members of the men's field hockey team: David Ames, Mark Gleghorne, Iain Lewers and Ian Sloan.\", \" The team also represents, and included representation from, the Crown dependencies, among which were Guernsey's Heather Watson and Carl Hester, and from the ten of the thirteen British Overseas Territories represented by the BOA rather than their own NOC, whose representatives include Turks and Caicos-born sprinter Delano Williams and Anguillan-born long jumper Shara Proctor\"]], ['James Brown (Elvis impersonator)', ['James \"The King\" Brown (born 1968) is a Belfast-born Elvis Presley tribute act known for his covers of songs done in the style of Elvis.', ' In the vein of \"songs that Elvis should have done,\" Brown performs songs like \"Whole Lotta Rosie\", originally by AC/DC, and \"Crazy Little Thing Called Love\", originally by Queen.']], ['Whole Lotta Trouble', ['\"Whole Lotta Trouble\" is a song by the American singer-songwriter Stevie Nicks.', ' The song was written by Nicks and Tom Petty and the Heartbreakers guitarist Mike Campbell.', ' The song became a minor hit in the United Kingdom, reached #62 on the charts.', ' In Ireland, The song gained much more attention, reached #22 on the charts.', ' The song was last performed by Nicks on August 29, 2000 where it was performed in San Diego, California.', ' The song was nominated for the Grammy Award for Best Rock Vocal Performance, Female.']], ['Jackie Woodburne', ['Jacqueline Anne \"Jackie\" Woodburne (born 5 February 1956) is a Northern Ireland born Australian actress best known for her television roles in soap operas \"Prisoner\" and \"Neighbours\".']], ['Whole Lotta Sole', [\"Whole Lotta Sole (known as Stand Off in North America) is a 2012 independent comedy film written and directed by Terry George and starring Brendan Fraser, David O'Hara, Colm Meaney, Yaya DaCosta and Martin McCann.\"]], ['Beauty and the Beast (Stevie Nicks song)', ['\"Beauty and the Beast\" is a song by the American singer-songwriter Stevie Nicks.', ' It is the final track on her second album \"The Wild Heart\", released in 1983.', ' It was later released in a live version from Nicks 1986 \"Rock a Little\" tour as a B-side to the UK single \"Whole Lotta Trouble\" in October 1989.', ' It also appears on two compilations: \"Timespace – The Best of Stevie Nicks\", released in 1991, and the boxset, \"Enchanted\", released in 1998.', ' A new studio version appears on her album, \"The Soundstage Sessions\", released in 2009.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.568\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab29cef554299449642c936', 'answer': '1861', 'question': 'In which year was the University where Joost Alois Businger is an emeritus professor established ?', 'supporting_facts': [['Joost Businger', 1], ['University of Washington', 0]], 'context': [['David S. G. Goodman', [\"David Stephen Gordon Goodman (born 1948) is Professor and Head of the Department of China Studies at Xi'an Jiaotong-Liverpool University in Suzhou.\", ' He is also Emeritus Professor of Chinese Politics at the University of Sydney and an Emeritus Professor at the University of Technology, Sydney.']], ['University of Washington', ['The University of Washington (commonly referred to as UW, simply Washington, or informally \"U-Dub\") is a large, public flagship research university in Seattle, Washington, established in 1861.']], ['Peter McLaren', ['Peter McLaren (born August 2, 1948) is Distinguished Professor in Critical Studies, College of Educational Studies, Chapman University, where he is Co-Director of the Paulo Freire Democratic Project and International Ambassador for Global Ethics and Social Justice.', ' He is also Emeritus Professor of Urban Education, University of California, Los Angeles, and Emeritus Professor of Educational Leadership, Miami University of Ohio.', ' He is also Honorary Director of Center for Critical Studies in Education in Northeast Normal University, Changchun, China.']], ['Barry Conyngham', ['Emeritus Professor Barry Ernest Conyngham, {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} , (born 27 August 1944) is an Australian composer and academic.', ' He has over seventy published works and over thirty recordings featuring his compositions, and his works have been premiered or performed in Australia, Japan, North and South America, the United Kingdom and Europe.', ' His output is largely for orchestra, ensemble or dramatic forces.', ' He is an Emeritus Professor of both the University of Wollongong and Southern Cross University.', ' He is currently Dean of the Faculty of the VCA and MCM at the University of Melbourne.']], ['Jay Blumler', ['Jay Blumler (born 1924) is an American-born theorist of communication and media.', ' He is now Emeritus Professor of Public Communication at the University of Leeds, and also Emeritus Professor of Journalism at the University of Maryland, having spent his early academic life largely in the UK.']], ['Cees Hamelink', ['Cees Jan Hamelink (born 14 September 1940) is a Dutch academic known for his work on communication, culture, and technology.', ' He is emeritus professor of international communications and emeritus professor of media at the University of Amsterdam; professor in management information and knowledge at the University of Aruba; and professor of media, religion and culture at the Vrije Universiteit Amsterdam.']], ['Colum Kenny', ['Colum Kenny is an author and emeritus professor at Dublin City University (DCU), in Dublin, Ireland.', ' Emeritus Professor, B.C.L., Barrister-at-Law, Ph.D.', ' Columnist for the \"Irish Times\".', ' Formerly chair of the Masters in Journalism programme at DCU.', ' School of Communications faculty 1982-2015.', ' Areas of special interest include media and culture, history and society.', ' A member of the Broadcasting Authority of Ireland 2010-2015 and of the Broadcasting Commission of Ireland/IRTC 1998-2003.', ' A former employee of RTE, he was a founding board member of the E.U. Media Desk in Ireland and is a council member of the Irish Legal History Society.', ' He was a member of the Media Mergers Advisory Group that reported to the Minister for Enterprise, Trade and Employment in 2008.', ' The author of many academic articles on cultural and media matters, he is also a member of the National Union of Journalists and a frequent contributor to media debates and a consultant on communications.', \" Awarded the DCU President's Award for Research in the Humanities and Social Sciences, 2004/5.\"]], ['Journal of Contemporary Asia', ['The Journal of Contemporary Asia (JCA) is a peer-reviewed academic journal in the field of Asian studies.', ' It was established in 1970 and is published quarterly by Routledge.', ' It is currently edited by Kevin Hewison (Emeritus Professor, University of North Carolina at Chapel Hill).', ' One of its founders, Peter Limqueco is editor emeritus.', ' The co-editors are Geoffrey Gunn (Emeritus Professor, Nagasaki University), Richard Westra (Nagoya University) and Toby Carroll (City University of Hong Kong).']], ['Donald Caspar', ['Donald L. D. Caspar (born January 8, 1927) is an American structural biologist (the very term he coined) known for his works on the structures of biological molecules, particularly of the tobacco mosaic virus.', ' He is an emeritus professor of biological science at the Institute of Molecular Biophysics, Florida State University, and an emeritus professor of biology at the Rosenstiel Basic Medical Sciences Research Center, Brandeis University.', ' He has made significant scientific contributions in virus biology, X-ray, neutron and electron diffraction, and protein plasticity.']], ['Joost Businger', ['Joost Alois Businger (born 29 March 1924) is a Dutch-American meteorologist.', ' He is an emeritus professor at the University of Washington.', ' Businger is best known for his work on atmospheric boundary layer (ABL).']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.569\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abb66c05542992ccd8e7f3e', 'answer': 'spot-fixing', 'question': 'What offence were opening batsman Khalid Latif and 5 other cricketers suspended for, in February 2017?', 'supporting_facts': [['2017 Pakistan Super League spot-fixing scandal', 1], ['Khalid Latif (cricketer)', 1], ['Khalid Latif (cricketer)', 2]], 'context': [['Ray Gripper', ['Raymond Arthur Gripper (born 7 July 1938), in Salisbury, Southern Rhodesia, was a cricketer.', ' He was a right-handed opening batsman and became a regular member of the Rhodesian side for 15 years starting in 1957–58, at one stage captaining them.', ' His highest score was an innings of 279 not out made against Orange Free State in 1967–68.', ' This remained a Currie Cup record for some years.', ' His son Trevor played Test cricket for Zimbabwe, also as an opening batsman.']], ['Khalid Latif (cricketer)', ['Khalid Latif (Urdu: \\u200e ), (born 4 November 1985 in Karachi) is a Pakistani cricketer.', ' A right-handed opening batsman, Latif captained Pakistan in the 2004 U-19 Cricket World Cup win and the 2010 Asian Games bronze medal win.', ' In 2017, the Pakistan Cricket Board banned Latif from all forms of cricket for five years, for his involvement in spot-fixing.']], ['Len Hutton', ['Sir Leonard Hutton (23 June 1916\\xa0– 6 September 1990) was an English cricketer who played as an opening batsman for Yorkshire from 1934 to 1955 and for England in 79 Test matches between 1937 and 1955. \"', 'Wisden Cricketers\\' Almanack\" described him as one of the greatest batsmen in the history of cricket.', ' He set a record in 1938 for the highest individual innings in a Test match in only his sixth Test appearance, scoring 364 runs against Australia, a milestone that stood for nearly 20 years (and remains an England Test record).', ' In 1952, he became the first professional cricketer of the 20th Century to captain England in Tests; under his captaincy England won the Ashes the following year for the first time in 19 years.', \" Following the Second World War, he was the mainstay of England's batting, and the team depended greatly on his success.\"]], ['Bill Ponsford', ['William Harold \"Bill\" Ponsford (19 October 1900\\xa0– 6 April 1991) was an Australian cricketer.', ' Usually playing as an opening batsman, he formed a successful and long-lived partnership opening the batting for Victoria and Australia with Bill Woodfull, his friend and state and national captain.', ' Ponsford is the only player to twice break the world record for the highest individual score in first-class cricket; Ponsford and Brian Lara are the only cricketers to twice score 400\\xa0runs in an innings.', \" Ponsford holds the Australian record for a partnership in Test cricket, set in 1934 in combination with Donald Bradman(451 for 2nd wicket)—the man who broke many of Ponsford's other individual records.In fact,he along with Don Bradman set the record for the highest partnership ever for any wicket in Test cricket history when playing in away soil (451 runs for the second wicket)\"]], ['2017 Pakistan Super League spot-fixing scandal', [\"The 2017 Pakistan Super League spot-fixing scandal arose in February 2017 when the Pakistan Cricket Board (PCB) suspended cricketers under its anti-corruption code in an ongoing investigation backed by International Cricket Council (ICC)'s Anti-Corruption and Security Unit on spot-fixing during the 2017 Pakistan Super League.\", ' The six cricketers suspended by the PCB are: Sharjeel Khan (on 10 February), Khalid Latif (on 10 February), Nasir Jamshed (on 13 February), Mohammad Irfan (on 14 March), Shahzaib Hasan (on 17 March) and Mohammad Nawaz (16 May).']], ['Tamim Iqbal', ['Tamim Iqbal Khan (Bengali: তামিম ইকবাল খান ; born 20 March 1989) is an international Bangladeshi cricketer and former Test captain of the team.Tamim is arguably the best batsman in Bangladesh.', ' Tamim made his One Day International debut in 2007 and played his first Test the following year.', \" A left-handed opening batsman, he is the Bangladeshi's most successful runscorer to date.\", ' Between December 2010 and September 2011 he was vice-captain of the national side.', ' Considered as the best ever opening batsman for Bangladesh, Tamim has set up centuries in all three formats of the game and is also the first Bangladeshi to score 10,000 international runs.']], ['Sidath Wettimuny', ['Sidath Wettimuny is a former Sri Lankan cricketer, who played Test cricket and One Day Internationals as an opening batsman from 1982 to 1987.', ' Wettimuny was a typical opening batsman in that he often played very defensively, grafting for his runs, and his ODI strike rate of 48 shows this quite clearly.']], ['Khalid Latif (imam)', ['Khalid Latif is Executive Director and Chaplain (Imam) for the Islamic Center at New York University (NYU).']], ['Roy Virgin', ['A right-handed opening batsman, Virgin had a mostly solid but unspectacular career in first-class cricket, except for two individual seasons, one for each of his two counties, during which he looked as good as any opening batsman in county cricket and was mentioned as a possible Test player.']], ['Angus Robson', ['Angus James Robson (born 19 February 1992 in Sydney) is an Australian cricketer who played for Leicestershire.', ' He is the brother of England and Middlesex opening batsman, Sam.', ' He has appeared in 26 first-class matches as a right-handed batsman who bowls leg breaks.', ' He was part of the Leicestershire side that completed a famous first victory in 3 years against Essex on 3 June 2015, playing a big role in the side as an opening batsman, scoring 120 and 71 in the game.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.569\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae6479a55429929b0807b1b', 'answer': '\"That Bizarre Girl\"', 'question': \"Jun Ji-hyun rose to fame after her as a girl in a film that's title means what?\", 'supporting_facts': [['Jun Ji-hyun', 0], ['Jun Ji-hyun', 1], ['My Sassy Girl', 0]], 'context': [['My Sassy Girl', ['My Sassy Girl (Korean: 엽기적인 그녀 ; literally, \"That Bizarre Girl\") is a 2001 South Korean romantic comedy film directed by Kwak Jae-yong, starring Jun Ji-hyun and Cha Tae-hyun.']], ['Il Mare', ['Il Mare (; lit.', ' \"time-transcending love\") is a 2000 South Korean film, starring Jun Ji-hyun and Lee Jung-jae, and directed by Lee Hyun-seung.', ' The title, \"Il Mare\", means \"The Sea\" in Italian, and is the name of the seaside house which is the setting of the story.', ' The two protagonists both live there two years apart in time, but are able to communicate through a mysterious mailbox.']], ['Happy Together (1999 TV series)', ['Happy Together () is a 1999 South Korean television series starring Lee Byung-hun, Song Seung-heon, Kim Ha-neul, Jo Min-su, and Jun Ji-hyun It aired on SBS from June 16 to August 5, 1999 on Wednesdays and Thursdays at 21:55 for 16 episodes.', ' Starring young actors who would go on to become Korean TV and film stars, the hit drama revolves around five children who were separated at the death of their parents, and the love, conflicts, and reconciliation that these siblings go through when they meet again as adults.']], ['Windstruck', ['Windstruck (; lit.', ' \"Let me introduce (you to) my girlfriend\") is a 2004 South Korean romantic comedy.', ' It stars Jun Ji-hyun, Jang Hyuk, and was directed by Kwak Jae-yong.', ' The film held its premiere in Hong Kong, attended by Jang and Jun, on 28 May 2004, being the first Korean film to do so.', ' It was released on June 3, 2004 by CJ Entertainment and ran at 123 minutes.']], ['Jun Ji-hyun', ['Jun Ji-hyun (born Wang Ji-hyun on 30 October 1981), also known as Gianna Jun, is a South Korean actress.', ' She rose to fame for her role as The Girl in the romantic comedy \"My Sassy Girl\" (2001), one of the highest-grossing Korean comedies of all time.', ' Other notable films include \"Il Mare\" (2000), \"Windstruck\" (2004), \"The Thieves\" (2012), \"The Berlin File\" (2013) and \"Assassination\" (2015).']], ['My Love from the Star', ['My Love from the Star (; literally \"You Who Came from the Stars\") is a South Korean television series starring Jun Ji-hyun, Kim Soo-hyun, Park Hae-jin and Yoo In-na in lead.', ' Written by Park Ji-eun, it is a romantic fantasy story about an alien who landed on Earth in the Joseon Dynasty and, 400 years later, falls in love with a top actress in the modern era.', \" It aired on SBS from December 18, 2013 to February 27, 2014 on Wednesdays and Thursdays at 22:00 for 21 episodes; the production company extended the original 20-episode run with one episode, due to high viewers' demand.\"]], ['The Berlin File', ['The Berlin File (; lit.', ' \"Berlin\") is a 2013 South Korean spy action thriller film written and directed by Ryoo Seung-wan.', ' Ha Jung-woo stars as a North Korean agent in Berlin who is betrayed and cut loose when a weapons deal is exposed.', ' Together with his wife, a translator at the North Korean embassy in Berlin played by Jun Ji-hyun, they try to escape being purged, with Ryoo Seung-bum and Han Suk-kyu playing North and South Korean operatives on their trail.']], ['White Valentine', ['White Valentine () is a 1999 Korean romantic film directed by Yang Yun-ho.', ' It stars Park Shin-yang with Jun Ji-hyun in her movie debut.']], ['Daisy (2006 film)', ['Daisy () is a 2006 film directed by Hong Kong filmmaker Andrew Lau of the \"Infernal Affairs\" trilogy.', ' \"Daisy\" is an urban romantic melodrama involving young painter Hye-young (Jun Ji-hyun), Interpol detective Jeong Woo (Lee Sung-jae), and professional hitman Park Yi (Jung Woo-sung).']], ['The Legend of the Blue Sea', ['The Legend of the Blue Sea () is a 2016-2017 South Korean television series starring Jun Ji-hyun and Lee Min-ho.', \" Inspired by a classic Joseon legend from Korea's first collection of unofficial historical tales about a fisherman who captures and releases a mermaid, this drama tells the love story of a con-artist and a mermaid who travels across the ocean to find him.\", ' It aired on SBS every Wednesday and Thursday at 22:00 (KST) started from 16 November 2016 until 25 January 2017.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.570\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac4f2e755429924173fb503', 'answer': 'Battle of Prome', 'question': 'Chiang Chungcheng was involved involved in what battle in 1942?', 'supporting_facts': [['Battle of Prome (1942)', 0], ['Chiang Kai-shek', 0]], 'context': [['Battle of Prome (1942)', ['The Battle of Prome (pyay-1942) was a battle in the Japanese conquest of Burma.', ' China\\'s Generalissimo, Chiang Kai-shek, believed \"As long as the British hold Prome, we hold Toungoo.\"']], ['Soong Mei-ling', ['Soong Mei-ling or Soong May-ling (; March 5, 1898 – October 23, 2003), also known as Madame Chiang Kai-shek or Madame Chiang, was a Chinese political figure who was First Lady of the Republic of China (ROC), the wife of Generalissimo and President Chiang Kai-shek.', ' Soong played a prominent role in the politics of the Republic of China and was the sister-in-law of Sun Yat-sen, the founder and the leader of the Republic of China.', ' She was active in the civic life of her country and held many honorary and active positions, including chairman of Fu Jen Catholic University.', ' During the Second Sino-Japanese War she rallied her people against the Japanese invasion and in 1942 conducted a speaking tour of the United States to gain support.', ' She was also the youngest and the last surviving of the three Soong sisters, and the only first lady during World War II (aside from Queen Elizabeth, 1900-2002) who lived into the 21st century.', ' Her life extended into three centuries.']], ['Chang Ya-juo', ['Chang Ya-jo (; 1913–1942; sometimes romanized as \"Chang Ya-juo\" or \"Chang Yaruo\") was the mistress of Chiang Ching-kuo () and bore twin sons for him, John Chiang () and Winston Chang () in 1942.', ' She met Chiang when she was working at a training camp for enlistees in the fight against Japan while he was serving as the head of Gannan Prefecture.']], [\"First Battle of Eora Creek – Templeton's Crossing\", [\"The First Battle of Eora Creek\\xa0– Templeton's Crossing was fought from 31 August 1942 to 5 September 1942.\", \" Forming part of the Kokoda Track campaign of the Second World War, the battle involved military forces from Australia, supported by the United States, fighting against Japanese troops from Major General Tomitaro Horii's South Seas Detachment who had landed in Papua in mid-1942, with the intent of capturing Port Moresby.\", ' The battle was one of three defensive actions fought by the Australians along the Kokoda Track.', ' The fighting resulted in the delay of the Japanese advance south, which allowed the Australians to withdraw to Efogi.', \" Eora Creek village and Templeton's Crossing was subsequently the site of a battle in late October 1942 as the Australian forces pursued the Japanese forces retiring back toward the north coast of Papua.\"]], ['Chiang Kai-shek', ['Chiang Kai-shek (October 31, 1887 – April 5, 1975), also romanized as Chiang Chieh-shih and known as Chiang Chungcheng, was a Chinese statesman, political and military leader who served as the leader of the Republic of China.']], ['Battle of Yiwu', ['The Battle of Yiwu (伊吾) is also called the Battle to Defend Yiwu (伊吾保卫战) by the Communist Party of China, and resulted in the communist victory.', ' After the local nationalist commanders in Xinjiang defected to the communist side, many nationalists loyal to Chiang Kai-shek refused to join the communists, and one detachment of these loyal nationalists decided to take the town of Yiwu to turn it into a guerrilla base in order to fight on until the eventual return of Chiang.']], ['John Chiang (Taiwan)', ['John Chiang or Chiang Hsiao-yen (; born March 1, 1942), formerly surnamed Chang (), is a Kuomintang politician in Taiwan.', ' He is the grandson of Chiang Kai-shek, former leader of the Republic of China.']], ['Campaign to Defend Siping', ['The Campaign to Defend Siping (四平保卫战) was a struggle between the Nationalists and the communists for the control of Siping during the Chinese Civil War in the post World War II era.', ' The nationalists have combined this campaign with the Battle of Siping as part of the battle, but this was rather misleading since the strategies for both sides were totally different from the strategies in this campaign and unrelated to each other, furthermore, the commanders for both sides in this campaign were completely different from the Battle of Siping.', ' More importantly, the nationalists in the Battle of Siping was in name only, because they were former nationalists (mostly warlords ostensively under nationalist reign) turned Japanese puppet regime forces who rejoined the nationalists after World War II, and the local bandits recruited by the nationalist administrators to fight off communists, since Chiang Kai-shek’s nationalist regime simply did not have the resource to rapidly deploy his forces into the region.', ' In fact, in the Battle of Siping, Chiang’s own force did not even participated in the fights.', ' This campaign was characterized by the fact that the supreme commanders of both sides had overestimated their strength and set unrealistic goals that could doom their troops in the field, but in both cases, the brilliant frontline commanders on both sides had successfully averted the potential catastrophes by convincing their respective supreme commanders to change their original decisions.']], ['Chungcheng', ['Zhongzheng or Chungcheng () is a common name for places, roads, schools or organizations in Chinese-speaking areas, though today predominantly in Taiwan.', ' The majority of these places and things are named after Chiang Chung-cheng, the preferred given name of Chiang Kai-shek.', ' As a result, when translating into English or other non-Chinese languages, it sometimes would be replaced by \"Chiang Kai-shek\" instead of simply by transliteration.']], ['Wang Kunlun', ['Wang Kunlun (1902–1985), birth name Wang Ruyu, was a Chinese politician who held high-profile positions, at different times, in both the Nationalist and Communist parties.', ' Born 1902 in Baoding, Hebei province to a wealthy household, he participated in the May Fourth Movement while studying at Peking University and became involved with Chinese revolutionaries, at one point meeting in person with Dr. Sun Yat-sen.', \" He joined the Nationalist party as a left-leaning member and served as Chief Secretary of the Political Department of the Headquarters of the National Revolutionary Army during the Northern Expedition, but became disillusioned with Chiang Kai-shek's leadership after Chiang initiated a major crackdown against Communists in April 1927.\", ' He subsequently joined the Communist Party in secret and used his political positions within the Nationalist government to aid the Communists.', ' He was among a group of members of the Kuomintang who broke away to form the Revolutionary Committee of the Kuomintang in 1948.', \" He would serve various government positions after the Communist victory, including vice-mayor of Beijing and vice-chairman of the National Committee of the Chinese People's Political Consultative Conference.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.571\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7f7ca25542994857a76747', 'answer': 'first', 'question': ' Keith Konrad Slettedahl appeared on which season of \"How I Met Your Mother\"?', 'supporting_facts': [['Keith Slettedahl', 0], ['Keith Slettedahl', 2], ['Best Prom Ever', 0]], 'context': [['The Mother (How I Met Your Mother)', ['Tracy McConnell, better known as \"The Mother\", is the title character from the CBS television sitcom \"How I Met Your Mother\".', ' The show, narrated by Future Ted, tells the story of how Ted Mosby met The Mother.', ' Tracy McConnell appears in 8 episodes from \"Lucky Penny\" to \"The Time Travelers\" as an unseen character; she was first seen fully in \"Something New\" and was promoted to a main character in season 9.', ' The Mother is played by Cristin Milioti.']], ['Best Prom Ever', ['\"Best Prom Ever\" is the 20th episode in the first season of the television series \"How I Met Your Mother\".', ' It originally aired in the United States on May 1, 2006.', ' It had the lowest recorded viewership for season 1 (7.24 million).']], ['Hurricane Keith', ['Hurricane Keith was an Atlantic hurricane in October 2000 that caused extensive damage in Central America, especially in Mexico and Belize.', \" It was the fifteenth tropical cyclone, eleventh named storm, and seventh hurricane of the that year's Atlantic hurricane season.\", ' Keith developed as a tropical depression from a tropical wave in the western Caribbean Sea on September\\xa028.', ' The depression gradually strengthened, and became Tropical Storm Keith on the following day.', ' As the storm tracked westward, it continued to intensify and was upgraded to a hurricane on September\\xa030.', ' Shortly thereafter, Keith began to rapidly deepen, and peaked as a Category\\xa04 hurricane less than 24\\xa0hours later.', ' Keith then began to meander erratically offshore of Belize, which significantly weakened the storm due to land interaction.', ' By late on October\\xa02, Keith made landfall in Ambergris Caye, Belize as a minimal hurricane.', ' It quickly weakened to a tropical storm, before another landfall occurred near Belize City early on the following day.', ' While moving inland over the Yucatán Peninsula, Keith weakened further, and was downgraded to a tropical depression before emerging into the Gulf of Mexico on October\\xa04.', ' Once in the Gulf of Mexico, Keith began to re-strengthen and was upgraded to a tropical storm later that day, and a hurricane on the following day.', ' By late on October\\xa05, Keith made its third and final landfall near Tampico, Tamaulipas, Mexico as a moderately strong Category 1 hurricane.', ' The storm quickly weakened inland and dissipated as a tropical cyclone by 24\\xa0hours after landfall.']], ['The 88', ['The 88 was an American pop rock band from Los Angeles, California.', \" The group consisted of Keith Slettedahl (vocals and guitar), Adam Merrin (keyboards and vocals), Anthony Zimmitti (drums and percussion), and Todd O'Keefe (bass and vocals).\", ' Meeting in high school in Calabasas, California, Adam Merrin and Keith Slettedahl formed the band in 2002.', \" The band's popularity grew from high-profile song placements in video games, films, commercials, and popular TV shows.\"]], ['Konrad KM-011', ['The Konrad KM-011 was a sports prototype built for Group C racing in the 1991 World Sportscar Championship season.', ' The car was part of a brief partnership between Konrad Motorsport and Lamborghini.']], ['Konrad (musician)', ['Konrad is the recording and stage name of Jeffrey Konrad, an American songwriter, record producer and musician.', ' Konrad runs the independent record label Radical Turf Records, a label that specializes in experimental, alternative and electronic music.', ' Konrad has appeared on half a dozen compilation albums for various record labels and has been played on North American college radio.']], ['List of EastEnders characters (2017)', ['The following is a list of characters that first appeared in the BBC soap opera \"EastEnders\" in 2017, by order of first appearance.', \" All characters are introduced by the show's executive producer Sean O'Connor or his temporary successor as creative director, John Yorke.\", ' The first character to be introduced was Keegan Baker (Zack Morris), a friend of Shakil Kazemi (Shaheen Jafargholi), followed by Emerald Fox (Doña Croll), the mother of Denise Fox (Diane Parish) and Kim Fox-Hubbard (Tameka Empson).', \" Madison Drake (Seraphina Beh), Alexandra D'Costa (Sydney Craven) and Travis Law-Hughes (Alex James-Phelps), three new teenage characters, were also introduced in January as well as their school teacher Mr Gethin Pryce (Cerith Flinn) and Hugo Browning (Simon Williams), the chairman of Weyland & Co.\", ' The following month, Preston Cooper (Martin Anzor), a student with whom Michelle Fowler (Jenna Russell) had an illegal relationship in the United States, and Konrad Topolski (Piotr Baumann), a love interest for Shirley Carter (Linda Henry), made their first appearances.']], ['How I Met Your Mother (season 9)', ['The ninth and final season of \"How I Met Your Mother\", an American sitcom created by Carter Bays and Craig Thomas, premiered on CBS on September 23, 2013, with two episodes, and concluded on March 31, 2014.', ' The show was renewed for the final season on December 21, 2012, after cast member Jason Segel changed his decision to leave the show after Season 8.', ' Cristin Milioti, who was revealed as \"The Mother\" in the Season 8 finale, was promoted to a series regular, the only time \"How I Met Your Mother\" added a new regular cast member.', ' Season 9 consists of 24 episodes, each running approximately 22 minutes in length.']], ['How I Met Your Music', ['How I Met Your Music is the name of two albums (the first being followed by the subtitle (Original Songs from the Hit Series \"How I Met Your Mother\"), the second being followed by Deluxe) composed of songs from the CBS television series \"How I Met Your Mother\", the first of which was released hours before the Season 8 premiere.', ' It features 20 songs that had appeared in the first seven series on the show and was released only digitally, originally through iTunes.', ' A second iteration, titled \"How I Met Your Music: Deluxe\" album, was released a year later.', ' It contains an entirely different play list.', \" Many critics have said that the albums reflect the series' consistently effective use of music.\"]], ['Keith Slettedahl', ['Keith Konrad Slettedahl (born August 9, 1973) is an American musician.', ' He is the singer and songwriter for Los Angeles pop rock group The 88.', ' In 2006, he appeared in the episode \"Best Prom Ever\" on the sitcom \"How I Met Your Mother\", playing a gig with his band on a High School-Prom.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.572\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8cddd1554299441c6b9f3a', 'answer': 'Japan and Hong Kong', 'question': 'In what 2 countries did both X-Large and A Bathing Ape have stores?', 'supporting_facts': [['X-Large', 2], ['A Bathing Ape', 3]], 'context': [['Cyprus–Saudi Arabia relations', ['Cypriot–Saudi Arabian relations are foreign relations between Cyprus and Saudi Arabia.', ' The two countries share membership of the United Nations.', ' Cyprus is represented to Saudi Arabia through its accredited honorary consulate in Jeddah.', ' Saudi Arabia is represented to Cyprus through its accredited embassy in Nicosia.', ' The political relations are close due to similarities between the 2 countries on historical, geographical and economical issues.']], ['I.T', ['I.T () is a Hong Kong fashion conglomerate founded in 1988.', ' It owns a number of Hong Kong brands which it retails as well as distributing European and Japanese brands such as French Connection and A Bathing Ape.', ' It has a large presence in Asia and several stores in a single mall in Richmond, British Columbia (in Canada).']], ['X-Large', ['X-Large is a clothing store/line founded in Los Angeles in 1991.', \" It soon became popular with urban youth and hip-hop artists (Beastie Boys' Michael Diamond was one of the company's original partners).\", ' It is very popular in Japan and Hong Kong, it is most famous for its Gorilla Logo.', ' According to I.T\\'s official website, which is a retail chain that carries X-Large branded items in Hong Kong, X-Large was the first street brand to use a gorilla or monkey as a logo even before \"A Bathing Ape in Lukewarm Water\".']], ['Río Negro (Central America)', ['Río Negro (] ) is a river that divides the countries of Honduras and Nicaragua along the Pacific coast.', ' Its path was substantially altered by Hurricane Mitch in October 1998.', ' It mostly runs through a very undeveloped jungle region of the 2 countries.']], ['A Bathing Ape', ['A Bathing Ape (ア・ベイジング・エイプ , A Beijingu Eipu ) (or BAPE) is a Japanese clothing brand founded by Nigo in Ura-Harajuku in 1993.', \" The brand specializes in men's, women's and children's lifestyle and street wear, running 19 stores in Japan, including Bape Stores, Bape Pirate Stores, Bape Kids Stores, Bapexclusive Aoyama, and Bapexclusive Kyoto.\", ' The Kyoto store also includes Bape Gallery, a space used for various events and art shows sponsored by Bape.', ' There are also stores located in Hong Kong, New York City, London, Taipei, China, Bangkok and Singapore.']], ['Exact Data', ['Exact Data provides direct marketing services with a focus in postal, email, and telephone solutions.', ' The company provides mailing lists and email marketing services designed to help companies acquire and retain customers.', ' Exact Data is based in Chicago, Illinois and operates in 2 countries.']], ['Russia–Singapore relations', ['Russia–Singapore relations (Russian: Российско-сингапурские отношения ) refers to the bilateral foreign relations between the two countries, Russia and Singapore.', ' Russia has an embassy in Singapore.', ' Singapore has an embassy in Moscow.', ' Both countries are full members of APEC.', ' Relations between the 2 countries have been described as \"excellent\", with Russia and Singapore sharing many common interests and enjoying close collaborations on many levels']], ['BAPE', ['BAPE (or A Bathing Ape) is a Japanese clothing company.']], ['Nigo', ['Nigo (ニゴー , \"Nigō\" , born on December 23, 1970) is a Japanese fashion designer, DJ, record producer and entrepreneur.', ' He is best known as the creator of the urban clothing line A Bathing Ape (Bape).', ' He is the DJ of the Japanese hip hop group Teriyaki Boyz.']], ['Cyprus–Malta relations', ['Cypriot–Maltese relations are foreign relations between Cyprus and Malta.', ' The two countries share membership of the European Union and Commonwealth of Nations.', ' Cyprus is represented to Malta through its accredited embassy in Rome (Italy).', ' Malta is represented to Cyprus through its accredited embassy in Athens (Greece).', ' The political relations are close due to similarities between the 2 countries (on historical, economical and regional).', ' By May 2004, the two island countries, along with the Baltic states , Slovenia and the Visegrad Group entered the European Union.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.573\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae22a08554299234fd0440b', 'answer': 'Kittie', 'question': 'Which Canadian heavy metal band is Safe  the second EP by ', 'supporting_facts': [['Safe (EP)', 0], ['Kittie', 0]], 'context': [['Metalized', ['Metalized is the debut album by the Canadian heavy metal band Sword.', ' It was released in 1986 by the Canadian indie label Aquarius Records.', ' The album samples many subgenres of the heavy metal genre, such as thrash metal on \"Outta Control\".']], ['Long Live Heavy Metal', ['Long Live Heavy Metal is the fifth and final studio album from the Canadian heavy metal band 3 Inches of Blood.', ' It is the second 3 Inches of Blood album to be released through Century Media and not feature Jamie Hooper on screaming vocals.']], ['Canadian heavy metal', ['Canadian heavy metal music has a long history.', ' Going back to the late 1960s, Canada has produced metal bands that have and continue to influence metal bands to this day.', ' In 1964, Toronto-based band The Sparrows was formed.', ' This band later changed their name to Steppenwolf and featured Canadians John Kay, Goldy McJohn and Jerry Edmonton.', ' Steppenwolf\\'s 1968 single \"Born to be Wild\" was the first use of the words \\'heavy metal\\' in a song\\'s lyric.', ' In 1970, Woodstock, Ontario based Warpig released their metal music debut, which, although never reaching mainstream success like fellow heavy metal bands Black Sabbath and Blue Cheer, has become a cult favourite within the Doom metal scene.']], ['Kittie', ['Kittie (stylized as KiTTiE) are a Canadian heavy metal band formed in London, Ontario in 1996.', ' They have released six studio albums, one video album, four extended plays, thirteen singles and thirteen music videos.', ' The band chose \"Kittie\" as their band name because the name \"seemed contradictory\".']], ['Unleash the Archers', ['Unleash The Archers is a Canadian heavy metal band from Victoria, BC, currently signed with Napalm Records.', ' The band plays a fusion of traditional heavy metal with power metal and melodic metal.']], ['Cam Pipes', ['Cam Pipes is a Canadian heavy metal musician, who performed as the lead vocalist and bassist in the Canadian heavy metal band 3 Inches of Blood.', ' He performs a falsetto vocal style reminiscent of Udo Dirkschneider and King Diamond.', ' Pipes was the only member of 3 Inches of Blood to have been featured on all of their albums, although he was not a founding member of the band.']], ['Safe (EP)', ['Safe is the second EP by Kittie, a Canadian heavy metal all-women band from London, Ontario.', ' It was released in 2002.', ' It is dedicated \"In Loving Memory of Dave Williams\".', ' The EP sold 25,000 units in the United States.', ' It received very little promotion, only appearing on 2 major rock/heavy metal magazines.']], ['Heavier Than Metal', ['Heavier Than Metal is an EP by Canadian heavy metal band Skull Fist.', ' The EP was recorded at Shred Studios in Toronto and then released on March 1, 2010.', ' Alison Thunderland (drums), and Sir Shred (lead guitar) joined Jackie Slaughter (lead vocals/guitar/bass) to record this EP.', ' It was well received in the metal community and that led Skull Fist to be signed to Noise Art Records.']], ['Anvil (band)', ['Anvil are a Canadian heavy metal band from Toronto, Ontario, formed in 1978.', ' The band consists of Steve \"Lips\" Kudlow (vocals, guitar), Robb Reiner (drums) and Chris Robertson (bass).', ' To date, the band has released sixteen studio albums, and has been cited as having influenced many notable heavy metal groups, including Megadeth, Slayer, Anthrax and Metallica.']], ['Boize (band)', ['Boize was a Canadian heavy metal and glam metal band based in Montreal, Quebec.', ' The band was formed in the spring of 1989 in Laval, Quebec when vocalist Perry Blainey responded to an advertisement placed in the Montreal Gazette newspaper by bassist and keyboardist Stéphane Fania and guitarist Robert Kourie.', ' Boize was associated with record label Aquarius Records and was managed by Canadian musician star and recording studio owner Bill Hill, under his music production company and management agency Bill Hill Productions.', ' The band also had a brief association with South American heavy metal legend Alvacast, when singer Carlos \"Charly\" Lopez joined Boize as new vocalist in the fall of 1992.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.573\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae4d2a55542990ba0bbb161', 'answer': 'major water deity', 'question': 'Giselle Cossard was known as Mother Giselle of what type of diety?', 'supporting_facts': [['Giselle Cossard', 0], ['Giselle Cossard', 1], ['Yemoja', 0]], 'context': [['Type A Kō-hyōteki-class submarine', ['The \"Type A Ko-hyoteki\" (甲標的甲型 , Kō-hyōteki kō-gata , Target \\'A\\', Type \\'A\\') class was a class of Japanese midget submarines (\"Ko-hyoteki\") used during World War II.', ' They had hull numbers but no names.', ' For simplicity, they are most often referred to by the hull number of the mother submarine.', ' Thus, the midget carried by \"I-16\"-class submarine was known as I-16\\'s boat, or \"I-16tou.\"']], ['Isabel Briggs Myers', ['Isabel Briggs Myers (October 18, 1897\\xa0– May 5, 1980) was an American author and co-creator of a personality inventory known as the Myers–Briggs Type Indicator (MBTI).', ' Briggs Myers created the MBTI with her mother, Katharine Cook Briggs.']], ['Tripura Sundari Temple', ['Tripura Sundari Temple is situated in the ancient Udaipur, about 55\\xa0km from Agartala, Tripura believed to be one of the holiest Hindu shrines in this part of the country.', ' Popularly known as Matabari, crowns in a small hillock and is served by the red-robed priests who traditionally, minister to the mother goddess Tripura Sundari.', ' Considered to be one of the 51 Shakti Peethas, consists of a square type sanctum of the typical Bengali hut.', \" It is believed that Sati's right foot fell here during Lord Shiva's Dance.\", ' The temple consist a square type sanctum with a conical dome.', ' It was constructed by Maharaja Dhanya Manikya in 1501A.', 'D, there are two identical images of the same deity inside the temple.', ' They are known as Tripura Sundari (5\\xa0feet high) and Chhotima (2\\xa0feet high) in Tripura.', \" The idol of Kali is worshiped at the temple of Tripura Sundari in the form of 'Soroshi'.\", ' One is made of kasti stone which is reddish black in colour.', ' It is believed that the idol was Chhotima was carried by king in battlefield.', ' This temple is also known as Kurma Pitha because it the temple premises resembles kurma i.e. tortoise.', ' Every year on Diwali, a famous Mela takes place near the temple which is visited by more than two lakhs pilgrims.']], ['Neonatal isoerythrolysis', ['Neonatal isoerythrolysis, also known as hemolytic icterus, is a disease most commonly seen in kittens and foals, but has also been reported in puppies.', ' In the kitten this is referred to as \"fading kitten syndrome.\"', ' It occurs when the mother has antibodies against the blood type of the newborn.']], ['Sweet Porridge', ['\"Sweet Porridge\", often known in English under the title of \"The Magic Porridge Pot\", is a folkloric German fairy tale recorded by the Brothers Grimm, as tale number 103 in \"Grimm\\'s Fairy Tales\", in the 19th century.', ' It is Aarne-Thompson type 565, the magic mill.', ' Other tales of this type include \"Why the Sea Is Salt\" and \"The Water Mother\".']], ['Giselle Cossard', ['Giselle Cossard Binon Omindarewa, (31 May 1923, Tangier - 21 January 2016, Duke of Caxias), Mãe-de-santo of Candomblé of Rio de Janeiro, was a French Brazilian anthropologist and writer.', ' She was also known as Mother Giselle of Yemoja, Daughter of Saint John of Goméia, Initiated for the Orisha Yemoja.']], ['Yemoja', ['Yemoja (Yoruba: \"Yemọja\" ) is a major water deity from the Yoruba religion.', ' She is an orisha and the mother of all orishas, having given birth to the 14 Yoruba gods and goddesses.', ' She is often syncretized with either Our Lady of Regla in the afrocuban diaspora or various other Virgin Mary figures of the Catholic Church, a practice that emerged during the era of the Trans-Atlantic slave trade.', ' Yemoja is motherly and strongly protective, and cares deeply for all her children, comforting them and cleansing them of sorrow.', ' She is said to be able to cure infertility in women, and cowrie shells represent her wealth.', ' She does not easily lose her temper, but when angered she can be quite destructive and violent, as the flood waters of turbulent rivers.']], ['Theotokos of Vladimir', ['The Theotokos of Vladimir (Greek: Θεοτόκος του Βλαντίμιρ ), also known as Our Lady of Vladimir, Vladimir Mother of God, or Virgin of Vladimir (Russian: Владимирская Икона Божией Матери ) is a medieval Byzantine icon of the Virgin and Child.', ' In 1169 Andrei Bogolyubsky sacked Kiev, and, after plundering the city, stole much religious artwork, including a Byzantine \"Mother of God\" icon which was transferred to Vladimir (for references see Yury Dolgorukiy and Andrey Bogolyubskiy).', ' It is one of the most venerated Orthodox icons and a fine and early example of the iconography of the \"Eleusa\" (tenderness) type, with the Christ child snuggling up to his mother\\'s cheek.', ' The \"Theotokos\" (Greek for Virgin Mary, literally meaning \"Birth-Giver of God\") is regarded as the holy protectress of Russia.', ' The icon is displayed in the Tretyakov Gallery, Moscow in a functioning church in the grounds of the museum.', ' Her feast day is June 23rd o.s. / July 6th n.s. Even more than most famous icons, the original has been copied repeatedly for centuries, and many copies have considerable artistic and religious significance of their own.']], ['Portuguese poetry', ['The beginnings of Portuguese poetry go back to the early 12th century, around the time when the County of Portugal separated from the medieval Kingdom of Galicia in the northwest of the Iberian Peninsula.', ' It was in this region that the ancestral language of both modern Portuguese and modern Galician, known today as Galician-Portuguese, was the common language of the people.', \" Like the troubadour culture in the Iberian Peninsula and the rest of Europe, Galician-Portuguese poets sang the love for a woman, which often turned into personal insults, as she had hurt her lover's pride.\", ' However, this region produced a specific type of song, known as \"cantigas de amigo\" (songs of a friend).', ' In these, the lyrical subject is always a woman (though the singer was male) talking about her friend (lover) from whom she has been separated - by war or other activities - as shown in the Reconquista.', ' They discuss the loneliness that the woman feels.', \" But some poems also project eroticism, or confess the lover's meeting in a secret place, often through a dialogue she has with her mother or with natural elements (such could be considered a custom adapted from the pagan peoples in the region).\", ' Epic poetry was also produced, as was common in Romantic medieval regions (\"Gesta de D. Afonso Henriques\", of unknown authorship).']], [\"Eve's pudding\", [\"Eve's pudding, also known as Mother Eve's pudding, is a type of traditional British pudding now made from apples and Victoria sponge cake mixture.\", ' The apples are allowed to stew at the bottom of the baking dish while the cake mixture cooks on top.', ' The name is a reference to the biblical Eve.', \" It is a simplified version of Duke of Cumberland's pudding.\", ' The earliest known version dates from 1824, predating baking powder, and therefore uses grated bread and shredded suet.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.575\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae7ac495542993210983eee', 'answer': 'What Ever Happened to Baby Jane?', 'question': 'What film came out first, All the Marbles or What Ever Happened to Baby Jane?', 'supporting_facts': [['...All the Marbles', 0], ['Robert Aldrich', 0], ['Robert Aldrich', 1]], 'context': [['David Cerda', ['David Cerda (born June 13, 1961, Hammond, Indiana) is an American performer and playwright based in Chicago, Illinois.', ' He is currently the artistic director for Hell In A Handbag Productions.', ' His campy, highly theatrical plays have made him an infamous icon within the Chicago theater scene.', ' He has written and appeared in a transgressive adaptation of \"Rudolph, the Red-Hosed Reindeer\", \"How ‘What Ever Happened to Baby Jane?’', ' Happened\" and POSEIDON!', ' An Upside-Down Musical which won the New York International Fringe Festival Best Ensemble Award.']], ['Dave Willock', ['Dave Willock (August 13, 1909 – November 12, 1990) was an American character actor.', ' Willock appeared in 181 films and television series from 1939 to 1989.', ' He is probably most familiar to modern audiences from his performance as Baby Jane Hudson\\'s father in the opening scenes of the cult classic \"What Ever Happened to Baby Jane?', '\" (1962).', ' He played seven different characters on CBS\\'s \"Green Acres\" with Eddie Albert and Eva Gabor, mostly portraying clerks or elevator operators.']], ['...All the Marbles', ['…All the Marbles (reissued as The California Dolls) is a 1981 comedy-drama film about the trials and travails of a female wrestling tag team and their manager.', ' It was directed by Robert Aldrich (his final film) and stars Peter Falk, Vicki Frederick and Laurene Landon.', ' The Pittsburgh Steeler hall of famer \"Mean\" Joe Greene plays himself.']], ['What Ever Happened to...', ['What Ever Happened to... is a 1991 American made-for-television thriller drama film directed by David Greene and adapted for the small screen by Brian Taggert, based on the novel \"What Ever Happened to Baby Jane?', '\" by Henry Farrell and the 1962 theatrical film of the same name.', ' It stars real-life sisters Lynn Redgrave as Baby Jane Hudson and Vanessa Redgrave as Blanche Hudson, in the roles previously played by Bette Davis and Joan Crawford in the 1962 adaptation.']], ['Robert Aldrich', ['Robert Burgess Aldrich (August 9, 1918 – December 5, 1983) was an American film director, writer and producer, notable for such films as \"Vera Cruz\" (1954), \"Kiss Me Deadly\" (1955), \"The Big Knife\" (1955), \"What Ever Happened to Baby Jane?', '\" (1962), \"Hush… Hush, Sweet Charlotte\" (1964), \"The Flight of the Phoenix\" (1965), \"The Dirty Dozen\" (1967) and \"The Longest Yard\" (1974).']], ['What Ever Happened to Baby Toto?', ['What Ever Happened to Baby Toto?', ' (Italian: \"Che fine ha fatto Totò Baby?\" )', ' is a 1964 Italian black comedy film written and directed by Ottavio Alessi.', ' It is a parody of Robert Aldrich\\'s \"What Ever Happened to Baby Jane?', '\".']], ['Psycho-biddy', ['Psycho-biddy is a colloquial term for a subgenre of the horror/thriller movie that features a formerly-glamorous older woman who has become mentally unbalanced and terrorizes those around her.', ' The genre officially began in 1962 with the film \"What Ever Happened to Baby Jane?', '\" (though it had some antecedents) and lasted through the mid-1970s.', ' It has also been referred to by the terms Grande Dame Guignol, hagsploitation and hag horror.', ' Renata Adler, in her \"The New York Times\" review for the 1968 film \"The Anniversary\", referred to the genre as \"the Terrifying Older Actress Filicidal Mummy genre.\"']], ['What Ever Happened to Baby Jane? (1962 film)', ['What Ever Happened to Baby Jane?', ' is a 1962 American psychological thriller–horror film produced and directed by Robert Aldrich, starring Bette Davis and Joan Crawford, about an aging former actress who holds her paraplegic sister captive in an old Hollywood mansion.', ' The screenplay by Lukas Heller is based on the 1960 novel of the same name by Henry Farrell.', \" Upon the film's release, it was met with widespread critical and box office acclaim and was later nominated for five Academy Awards, winning one for Best Costume Design, Black and White.\"]], ['Baby Jane Hudson', ['Baby Jane Hudson is a fictional character and the antagonist of Henry Farrell\\'s 1960 novel \"What Ever Happened to Baby Jane?', '\" She was portrayed by Bette Davis in the 1962 film adaptation and by Lynn Redgrave in the 1991 made-for-TV remake.', ' The 1962 production is the better-known, with Bette Davis earning an Academy Award nomination for her performance.', ' The character is portrayed by Susan Sarandon,who plays Bette Davis, in the TV anthology \"Feud: Bette and Joan\" aired in 2017.']], ['Debbie Burton', ['Debbie Burton was an American singer.', ' She is best known for dubbing the singing voice of the young Baby Jane Hudson (played by child actress Julie Allred) in the 1962 film \"What Ever Happened to Baby Jane?', '\", singing the song \"I\\'ve Written a Letter to Daddy\".', ' Burton also sang a duet with Bette Davis, the rock and roll song \"What Ever Happened to Baby Jane?\"', ', written by Frank DeVol and Lukas Heller.', ' It was released as a promotional single, with Burton\\'s rendition of \"I\\'ve Written a Letter to Daddy\" on the flipside.', ' An instrumental version of \"What Ever Happened to Baby Jane?\"', ' can be heard in the movie.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.575\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a721e0e55429971e9dc928d', 'answer': 'Duke University', 'question': 'which university was founded first Duke University or Tufts University?', 'supporting_facts': [['Duke University', 0], ['Duke University', 1], ['Tufts University', 1]], 'context': [['Frederick M. Ellis', ['Frederick Melvin \"Fish\" Ellis (February 26, 1906 – July 19, 1967) was an American sportsman who has played football, basketball, baseball, and track.', ' He was also an athletics coach, administrator, and university professor.', ' He served as the head football coach at Tufts University from 1946 to 1953, compiling a record of 25–34–6.', ' Ellis was also the head basketball coach at Tufts from 1946 to 1953, tallying a mark of 74–75.', \" He is the namesake of Tufts University's home football field, Ellis Oval.\"]], ['Tufts University', ['Tufts University is a private research university incorporated in the municipality of Medford, Massachusetts, United States.', ' Tufts College was founded in 1852 by Christian Universalists who worked for years to open a nonsectarian institution of higher learning.', ' Charles Tufts donated the land for the campus on Walnut Hill, the highest point in Medford, saying that he wanted to set a \"light on the hill\".', ' The name was changed to Tufts University in 1954, although the corporate name remains \"the Trustees of Tufts College\".', ' For more than a century, Tufts was a small New England liberal arts college until its transformation into a larger research university in the 1970s.', ' Tufts is a charter member of the New England Small College Athletic Conference (NESCAC).', ' In 2017, the university accepted 14.8% of undergraduate applicants from a pool of 21,101.', ' In 2016, it was ranked 27th nationally and 156th internationally by \"U.S. News & World Report\".']], ['History of Tufts University', ['The history of Tufts University, originally Tufts College, can be traced back to 1847 when the Universalist Church set up convention for the creation of a university for the parish.', ' In 1852, the college was established when Boston businessman Charles Tufts donated 20 acres of land to the church to establish the college.', ' It is the second oldest college that was founded in the Boston area.', ' .', ' During the 19th century the college grew.', ' The official college seal, bearing the motto Pax et Lux (Peace and Light) was adopted in 1857.', ' The school colors of brown and blue were selected in 1876.', \" Tufts' mascot became Jumbo when P.T. Barnum gave a natural history museum to the university.\"]], ['Tufts OpenCourseWare', ['The Tufts OpenCourseWare (OCW) project, is a web-based publication of educational material from a number of Tufts University courses, providing open sharing of free, searchable, high-quality course content to educators, students, and self-learners throughout the global community.', ' The Tufts OCW initiative encourages the publication and free exchange of course materials on the World Wide Web.', ' First launched in June 2005, Tufts OCW provides materials with strong representation from Tufts’ health sciences schools, some of which are equivalent to textbooks in depth.', ' All materials on the Tufts OCW site are accessible at any time, free of charge.', ' As Tufts OCW is not a distance learning program, no registration, applications, prerequisites, or fees are required and no credit is granted.']], ['The Tufts Observer', ['The Tufts Observer, founded as the \"Tufts Weekly\", is an undergraduate student newsmagazine published at Tufts University.', \" First published in 1895 Tufts' first student newspaper, the Observer is the oldest student publication on campus.\", ' The Tufts Weekly was renamed the Tufts Observer in 1969.', ' Observer staff currently work out of the Media Advocacy Board (MAB) Lab, located on the second floor of Curtis Hall on College Avenue.']], ['The Zamboni (magazine)', ['The Zamboni is a student-run humor publication at Tufts University.', ' It was founded in 1989 and comes out with six issues per year, or once per month.', ' It contains satirical articles (such as fake news briefs, interviews, and op-ed pieces), cartoons, and photos.', ' It is known as \"Tufts University\\'s Only Intentionally Funny Magazine\" and its motto is \"Cowering Behind the First Amendment Since 1989.\"', ' \"The Zamboni\" is fully funded by the Student Activities Fee as allocated by the Tufts Senate.']], ['Tufts University School of Medicine', ['The Tufts University School of Medicine is one of the eight schools that constitute Tufts University.', ' The \"Times Higher Education (THE)\" and the \"Academic Ranking of World Universities (ARWU)\" consistently rank Tufts among the world\\'s best medical research institutions for clinical medicine.', \" Located on the university's health sciences campus in downtown Boston, Massachusetts, the medical school has clinical affiliations with thousands of doctors and researchers in the United States and around the world, as well as at its affiliated hospitals in both Massachusetts (including Tufts Medical Center, St. Elizabeth's Medical Center, Lahey Hospital and Medical Center and Baystate Medical Center), and Maine (Maine Medical Center).\", ' According to Thomson Reuters\\' \"Science Watch\", Tufts University School of Medicine\\'s research impact rates sixth among U.S medical schools for its overall medical research and within the top 5 for specialized research areas such as chronic obstructive pulmonary disorder, urology, cholera, public health & health care science, and pediatrics.', ' In addition, Tufts University School of Medicine is ranked 44th in research and 38th in primary care according to \"U.S. News & World Report\".']], ['Pedram Hamrah', ['Pedram Hamrah is an ophthalmologist and immunologist.', ' He obtained his M.D. from the University of Cologne, Germany.', ' In 2002, together with Reza Dana and Ying Liu, he was the first to discover the presence of and characterize resident antigen-presenting cells in the central cornea.', ' Hamrah is currently Director of the Center for Translational Ocular Immunology and Director of Anterior Segment Imaging of the Boston Image Reading Center at the New England Eye Center, Department of Ophthalmology Tufts Medical Center, Tufts University School of Medicine.', ' In addition he is on the faculty of the Programs of Immunology and Neuroscience at the Sackler School of Graduate Biomedical Sciences at Tufts University.', \" He was a faculty member in the laboratory of Ulrich von Andrian at Harvard's Immune Disease Institute from 2008 to 2012.\"]], ['Duke University', ['Duke University is a private research university located in Durham, North Carolina.', ' Founded by Methodists and Quakers in the present-day town of Trinity in 1838, the school moved to Durham in 1892.', ' In 1924, tobacco and electric power industrialist James Buchanan Duke established The Duke Endowment, at which time the institution changed its name to honor his deceased father, Washington Duke.']], ['Tufts University School of Dental Medicine', ['Tufts University School of Dental Medicine (TUSDM) is a private, American dental school located in the Chinatown neighborhood of Boston, Massachusetts, and is connected to Tufts Medical Center.', ' It is one of the 8 graduate schools that comprise Tufts University.', ' Founded in 1868 as Boston Dental College by Dr. Isaac J. Wetherbee, the university is the second oldest dental school in the city, and one of the oldest in the country.', ' As of 2013, Tufts is the second largest dental school in the United States, with a class size of approximately 190 students per class.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.576\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae7909f5542994a481bbda6', 'answer': 'Spyker F1', 'question': 'Which of the teams for which Colin Kolles is the former team principal and managing director was sold and renamed Force India at the end of the 2007?', 'supporting_facts': [['Colin Kolles', 0], ['Spyker F1', 0], ['Spyker F1', 2]], 'context': [['ByKolles Racing', ['ByKolles Racing, formally known as Kodewa GmbH & Co.', ' KG, is an auto racing team based in Greding, Germany.', ' Founded in 2000 by Romulus Kolles and his son Colin Kolles as Kolles Racing, the company initially participated in German Formula 3 before moving to the F3 Euro Series from 2003 to 2005.', ' Colin left the team to become director of the Jordan Grand Prix Formula One team at the start of the 2005 season, a position he held until 2009.', ' With Colin away, Romulus moved the team to the Deutsche Tourenwagen Masters series with Audi under the sponsorship title Futurecom TME.', ' The Kolles team shifted their interest to sports car racing by participating in the Le Mans Series and later the Intercontinental Le Mans Cup, again with customer Audi prototypes.', \" Colin returned to Formula One to helm the HRT F1 team in 2010, with Kodewa's workshop in Greding serving as a base of operations for the new team before HRT's new owners chose to release Kolles from the team and move their operations to Spain in 2012.\", ' Kodewa participated in the 2013 FIA World Endurance Championship with the backing of Lotus Cars under the title Lotus LMP2.', ' In 2014 they lost their Lotus backing and rebranded as ByKolles Racing with a new LMP1 prototype, the CLM P1/01.']], ['Toyota TF110', ['The Toyota TF110 was an un-raced, prototype Formula One car designed by Toyota Racing for the 2010 Formula One season.', ' The car had been designed, and two chassis produced before Toyota officially decided to pull out of Formula One at the end of the 2009 Formula One season.', ' One chassis was damaged by former team principal John Howett, while the other was used for a shakedown test.', ' Various teams attempted to purchase the chassis.']], ['Monisha Kaltenborn', ['Monisha Kaltenborn ( Narang; born 10 May 1971) is the former team principal of the Sauber Formula One team and held a 33.3% stake in the outfit until it was taken over by Longbow Finance S.A. in July 2016.', \" She has also been the team's chief executive officer from January 2010.\", ' She was the first female team principal in Formula One.']], ['Spyker F8-VII', ['The Spyker F8-VII (subsequently known as the Force India VJM01) was a Formula One car, constructed by Spyker F1 that competed in the 2007 Formula One World Championship.', ' A \"B Specification\" car named the Spyker F8-VIIB was launched at the Italian Grand Prix and used for the remainder of the 2007 season.', ' The engine of Spyker F8-VII car was Ferrari 056 despite the team opted for 2006-spec engine due to cost reasons.', ' For the 2008 World Championship, Force India used a slightly modified version of the F8-VIIB, called the Force India VJM01 named after team owners Vijay Mallya, Jan Mol and Michiel Mol.', ' The VJM01 used 2007-spec Ferrari 056 engines instead of 2008-spec.', ' The F8-VII was the only car constructed by Spyker F1 in their own right after their take-over from the struggling Midland F1 team part way through 2006.']], ['Midland F1 Racing', ['Midland F1 Racing (also known as MF1 Racing) was a Formula One constructor and racing team.', ' It competed in the 2006 Formula One season with drivers Christijan Albers and Tiago Monteiro.', ' The team was created by the renaming of Jordan Grand Prix after its purchase by Canadian businessman, and owner of the Midland Group, Alex Shnaider.', \" The team was registered as the first Russian Formula One team, reflecting Shnaider's roots, although it continued to be based in the United Kingdom, at Jordan's Silverstone factory.\", ' Towards the end of the 2006 season, the team was sold to Spyker Cars N.V.; the team raced in its last three Grands Prix under the official name \"Spyker MF1 Racing\".', ' In 2007, the team competed as Spyker F1, and in 2008 was sold to Indian businessman Vijay Mallya and was renamed Force India F1.']], ['Frédéric Vasseur', ['Frédéric Vasseur (born in 1968 in Draveil, Ile-de-France) is a French motor sport engineer and manager with a long career managing Formula-series teams.', ' He is currently the Managing Director & CEO as well as Team Principal of Swiss based Sauber Motorsport AG.']], ['John Booth (motor racing)', ['John Alfred Booth (born 18 December 1954 in Rotherham, England) is the current Director of Racing at Scuderia Toro Rosso.', ' He is the former team principal of the Virgin/Marussia Formula One team.', \" He was initially the team's sporting director, but took over the role of team principal from Alex Tai less than one month after the team's launch.\"]], ['Alex Tai', ['Alexander Mark \"Alex\" Tai (born October 22, 1966) is the current team principal of the Virgin Racing Formula E team and former team principal of Virgin Racing Formula One team.']], ['Colin Kolles', ['Colin Kolles (born Călin Colesnic 13 December 1967 in Timişoara, Romania) is the former team principal and managing director of the Hispania Racing F1 Team, previously holding a similar position at the team known under the names Jordan, Midland, Spyker and Force India from 2005 to 2008.', ' He was an advisor to Caterham F1 and had a part in the unsuccessful Forza Rossa Racing project.']], ['Spyker F1', ['The Spyker F1 Team, known as the Etihad Aldar Spyker F1 Team for sponsorship reasons was a Formula One team that competed in the 2007 Formula One World Championship, and was created by Spyker Cars after their buyout of the short-lived Midland F1 (formerly Jordan Grand Prix) team.', ' The change to the Spyker name was accompanied by a switch in racing livery from the red and white previously used by Midland, to an orange and silver scheme—already seen on the Spyker Spyder GT2-R—orange being the national colour and the auto racing colour of the Netherlands.', ' At the end of the 2007 season the team was sold and renamed Force India.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.577\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac29aa655429967731025b2', 'answer': '26,000', 'question': 'Eduard Schweizer teaches at a German university with over how many students? ', 'supporting_facts': [['Eduard Schweizer', 0], ['University of Zurich', 0]], 'context': [['University of Zurich', ['The University of Zurich (UZH, German: \"Universität Zürich\" ), located in the city of Zürich, is the largest university in Switzerland, with over 26,000 students.', ' It was founded in 1833 from the existing colleges of theology, law, medicine and a new faculty of philosophy.']], ['BMVA Summer School', ['BMVA Summer School is an annual summer school on computer vision, organised by the British Machine Vision Association and Society for Pattern Recognition (BMVA).', ' The course is residential, usually held over five days, and consists of lectures and practicals in topics in image processing, computer vision, pattern recognition.', ' It is intended that the course will complement and extend the material in existing technical courses that many students/researchers will encounter in their early stage of postgraduate training or caeeers.', ' It aims to broaden awareness of knowledge and techniques in Vision, Image Computing and Pattern Recognition, and to develop appropriate research skills, and for students to interact with their peers, and to make contacts among those who will be the active researchers of their own generation.', ' It is open to students from both UK and non-UK universities.', ' The registration fees vary based on time of registration and are in general slightly higher for non-UK students.', ' The summer school has been hosted locally by various universities in UK that carry out Computer Vision research, e.g., Kingston University, the University of Manchester and Swansea University.']], ['University Cooperative Housing Association', ['University Cooperative Housing Association (UCHA) is a student housing cooperative in Westwood, Los Angeles serving the University of California, Los Angeles (UCLA) campus.', ' Approximately 400 students live there and in addition to housing UCLA students, UCHA offers housing to students of any college, including the UCLA Extension and Santa Monica College.', ' UCHA operates three buildings, Hardman-Hansen Hall, Essene Hall, and Robison Hall, the latter being a renovated version of the Landfair Apartments and cultural landmark designed by Richard Neutra.', ' Jim Morrison, of The Doors, purportedly lived at UCHA during his time at UCLA.', ' Green Day and Margaret Cho performed at UCHA in the early 1990s.', ' In addition to the UCLA campus, Hardman-Hansen and Robison Halls were used as filming locations for the 1982 horror film, The Dorm That Dripped Blood.', \" Many students of China's Lost Generation studying at UCLA reside at UCHA.\"]], ['Eduard Schweizer', ['Eduard Schweizer (1913-2006) was a Swiss New Testament scholar who taught at the University of Zurich for an extended period.', ' He won the Burkitt Medal for Biblical Studies in 1996.']], ['National High School Debate League of China', ['The National High School Debate League of China, or simply NHSDLC, is an English-language high school debate league serving Mainland China.', ' It uses the Public Forum debate format.', ' Each year, the NHSDLC sees around 50,000 students participate in its debate workshops and around 12,000 students participate in its regional or national tournaments that it hosts in more than 33 cities in China.', ' According to The Economist, many students believe participating will help their application to a Western university.', \" It was founded in 2012, and it hosted one of China's first ever English-language high school national debate tournaments for local students at Peking University in May 2013.\", ' Each year, its national debate championship hosted in Beijing attracts 450 students from around China.', ' NHSDLC is partnered with Harvard College Mentors for Urban Debate, Penn for Youth Debate, the Chicago Debate Society, the Yale Debate Association, Sunrise International Education, and the Stanford Youth Debate Initiative.']], ['Donald B. Fullerton', ['Donald B. Fullerton (July 6, 1892\\xa0– April 9, 1985) was a Christian missionary and teacher who founded the Princeton Evangelical Fellowship and served with it from 1931 until 1980.', ' He was noted for convincing many students at Princeton University of the truth of the Christian faith.', ' Arthur Glasser also credited his conversion to Dr. Fullerton, through hearing him speak at the Keswick Bible Conference.', ' In addition to his evangelistic efforts, Dr. Fullerton was a major spiritual influence on many students including Paul Pressler, a major figure in the Conservative resurgence of the Southern Baptist Convention, and the noted Reformed theologian John Frame.', ' He was a member of the Princeton University Class of 1913 and received an honorary Doctorate of Ministry from Grace Theological Seminary.']], ['Matthias Eduard Schweizer', ['Matthias Eduard Schweizer (8 August 1818 – 23 October 1860) was a Swiss chemist.']], ['Port Moody Secondary School', ['Port Moody Secondary School is a public coeducational high school located in Port Moody, British Columbia.', ' The school is notable for offering the International Baccalaureate Program and the Career Preparation Program to its students, which many students travel from other districts to participate in.', ' There are approximately 400 students in the pre-International Baccalaureate Diploma programme and the International Baccalaureate diploma programme tracks.', \" Port Moody Secondary is widely known in the area for sending an impressive number of students to the world's most selectivities universities.\", ' In the past three years, students have matriculated to schools such as: Harvard University, Princeton University, University of Chicago, University of Pennsylvania, Cornell University, UC Berkeley and Dartmouth College.', ' Port Moody serves grades nine through twelve and currently has an enrollment of 1,312 students.', ' The school is respected for its academics, visual arts, musical arts and athletic programs.']], [\"Pennsylvania Governor's School for the Sciences\", [\"The Pennsylvania Governor's School for the Sciences (PGSS) is one of the Pennsylvania Governor's Schools of Excellence, a group of five-week summer programs for gifted high school students in the state of Pennsylvania.\", ' Carnegie Mellon University in Pittsburgh has hosted the program since its inception in 1982.', ' Most recently, it has been directed by Physics Professor Dr. Barry Luokkala.', ' Participants are required to be Pennsylvania high school students between their junior and senior years and are required to live in the dormitories for the full five weeks of the program.', ' Admission is very competitive - approximately 500 of the most scientifically gifted students in the state compete for 56 to 60 slots in the program.', \" The aim of PGSS is to promote interest in science rather than to advance students' knowledge in a specific area.\", ' The curriculum includes five \"core\" courses in Biology, Chemistry, Computer Science, Mathematics and Physics, and numerous electives.', ' In addition to taking classes, students are required to participate in a lab course and a research-style team project.', ' The emphasis is on cooperation, rather than competition - students are encouraged to both collaborate with other students on academic work and to interact socially.', ' The Residence Life staff provides a number of structured social events to foster friendship and teamwork.', ' There is at least one event per day and is advertised on the social calendar in the dorm lobby.', ' For many students, the social development gained from the program rivals the scientific knowledge they acquire.', ' The students leave the program with a strong bond; most attend an organized reunion the following year after the 4th week of the program.']], ['KJSCE Symphony', ['Symphony, the annual cultural festival of K. J. Somaiya College of Engineering, has created its name and popularity among Engineering and Management institutes far and wide for the last decade.', ' Every year many students from various institutes be a part of this festival.', ' The main aim is to promote, encourage and exhibit the talents of the students on a common platform and create interest in the classical, vocal and instrumental music.', ' Symphony hosts more than 9000 students every year.', ' Symphony has been graced by artists of the magnitude of Pt.', ' Hariprasad Chaurasia, Pt ShivKumar Sharma, Louis Banks, Hariharan, Indus Creed, Parikrama, KK, Bombay Vikings, Taufiq Qureshi, Dagar, Suraj Jagan, and Ustad Zakir Hussain.', ' The event also has a social touch to propagate a message relevant to the times like AIDS awareness, etc.', ' There have also been Auto Shows and an Army display at Symphony.', ' The organization is done by students which is also a time for building strong camaraderie and teamwork.', ' Many students look back fondly at the memories gathered during this phase of their lives.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.577\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab2771e554299340b5254d8', 'answer': '29 September 2014', 'question': 'Said Fazal Akbar was appointed Governor by the President of Afghanistan that served until what day?', 'supporting_facts': [['Said Fazal Akbar', 1], ['Hamid Karzai', 0]], 'context': [['Luciano Valero', ['Luciano Valero is a Venezuelan politician.', ' He was the appointed Governor of Barinas from 1959 to 1964, and again from 1969 to 1973.', ' He was appointed Governor of the Federal District of Venezuela by President Luis Herrera Campins in the early 1980s.', ' He also served in the cabinet of Luis Herrera Campins, as Minister of Agriculture (1979–1981) and Minister of Interior and Justice (1982–1984).']], ['List of female Indian governors', ['In India, a governor is the constitutional head of each of the twenty-nine states.', \" The governor is appointed by the President of India for a term of five years, and holds office at the President's pleasure.\", ' The governor is \"de jure\" head of the state government; all its executive actions are taken in the governor\\'s name.', ' However, the governor must act on the advice of the popularly elected council of ministers, headed by the chief minister, who thus hold \"de facto\" executive authority at the state-level.', \" The Constitution of India also empowers the governor to act upon his or her own discretion, such as the ability to appoint or dismiss a ministry, recommend President's rule, or reserve bills for the President's assent.\", ' Over the years, the exercise of these discretionary powers have given rise to conflict between the elected chief minister and the central government–appointed governor.', ' The union territories of Andaman and Nicobar, Delhi and Puducherry are headed by lieutenant-governors.']], ['James R. Beverley', ['James Rumsey Beverley (June 15, 1894 – June 17, 1967) was a United States lawyer and politician, appointed as Attorney General of Puerto Rico, serving 1927-1932.', ' During this period, he was appointed as acting governor of Puerto Rico in 1929 and in January 1932, he was appointed Governor by President Herbert Hoover and served through 1933.', ' He was the only non-Puerto Rican appointee of 15 from 1900 to 1952 who could speak Spanish before going there.']], ['François-Xavier Donzelot', ['Baron François-Xavier Donzelot (7 January 1764, Mamirolle – 11 June 1843) was a French general and a Governor of the Ionian Islands and Martinique.', ' He was the son of François Donzelot and Jeanne–Baptiste Maire and had a brother named Joseph.', ' He became a general of the French army in March 1801.', ' Months later, he signed the surrender of Egypt to British forces.', \" He then returned to France where he served in various high-echelon positions in Napoleon's army.\", ' Subsequently, he was appointed to serve as the head of the French garrison in Corfu and the Ionian Islands from 1807 to 1814.', ' As governor, he resided in Corfu, where his gentle demeanour and mild manners made him popular with the Corfiotes.', ' In 1808, he was named Baron of the Empire.', \" In 1815, he was a divisional commander of Napoleon's forces at the Battle of Waterloo, during the 100-day return of Napoleon.\", ' After the defeat at Waterloo, he lost his position and did not work until 1817 when he was appointed governor of Martinique.']], ['Said Fazal Akbar', ['Said Fazal Akbar was the first Governor of Kunar province in Afghanistan after the fall of the Taliban in 2001.', ' He was an ex-Mujahideen who was running a clothing store in Oakland, California when he was appointed Governor by President Hamid Karzai.', ' His governorship is chronicled in the book by his son, Hyder Akbar, \"Come Back to Afghanistan, A California Teenager\\'s Story\"']], ['Theodore S. Parvin', ['Theodore Sutton Parvin was born on the 15th of January, 1817, in Cumberland County, New Jersey.', ' In 1833 he graduated at Woodworth College, Ohio, and began the study of law, graduating at the Cincinnati Law School in 1837.', ' In 1838 Robert Lucas, who had been appointed Governor of the new Territory of Iowa, selected Mr. Parvin for his private secretary.', ' He accompanied the Governor to Burlington where he was appointed to take charge of the Territorial library.', ' In 1839 Mr. Parvin was appointed District Attorney of the middle District and removed to Bloomington.', ' He served three terms as probate judge.', ' In 1844 he rendered Iowa an enduring service by cooperating with Enoch W. Eastman and Frederick D. Mills in defeating the Constitution which proposed to deprive the State of the counties of the Missouri slope.', ' Upon the organization of the United States District Court in 1846 Mr. Parvin was appointed clerk, a position he held ten years.', ' In 1857 he was nominated for Register of the State Land Office by the Democrats and, notwithstanding the Republican majority of more than 2,000 in the election for Governor the same year, Mr. Parvin was elected.', ' He was one of the first trustees of the State University and was for ten years professor of natural science in that institution.', ' He was one of the organizers of the State Historical Society and served several years as its secretary and as editor of the \"Annals of Iowa\", an historical magazine published by the society.', ' Mr. Parvin made large contributions to the library, newspaper files and general collections of that Society, and for more than thirty years was one of the most valued writers of historical and biographical articles for the \"Annals of Iowa\" and the \"Historical Record\".', ' Having been one of the first officials of the Territory and long associated with its public affairs, personally acquainted with prominent men of all parties for more than sixty years, Mr. Parvin was long regarded the highest authority on Iowa history and biography.', ' He was one of the founders of the Masonic Order of Iowa and has been Grand Master and Grand Secretary of the Grand Lodge of the State many years.', ' In his capacity as secretary he collected at their building at Cedar Rapids the most extensive Masonic library in the world.', ' He also collected and donated to the library a more complete collection of Iowa books and rare documents than is possessed by any other library.', \" Mr. Parvin's contributions of early Iowa newspapers, legislative journals and session laws, long out of print and other rare publications to the State and Historical libraries have been continuous and exceedingly valuable.\", \" He was one of the most valued members of the Pioneer Lawmakers' Association and his historical contributions to that organization have been of great value.\", ' His writings and addresses on historical subjects relating to Iowa for half a century would fill several volumes.', ' He died at his home at Cedar Rapids, June 28, 1901.']], ['Hamid Karzai', ['Hamid Karzai (Pashto/Dari: \\u200e \\u200e حامد کرزی; born 24 December 1957) served as President of Afghanistan for almost ten years, from 7 December 2004 to 29 September 2014.', \" He comes from a politically active family; Karzai's father, uncle and grandfather were all active in Afghan politics and government.\", ' Karzai and his father before him, Abdul Ahad Karzai, were each head of the Popalzai tribe of the Durrani tribal confederation.']], ['Joseph Flores (Guamanian politician)', ['Joseph F. Flores (August 12, 1900 – December 18, 1981) was the fourth civilian appointed Governor of Guam, and was the first Chamorro to hold the office.', ' He also founded the island\\'s first locally owned newspaper, the \"Guam Daily News\" (later becoming the Pacific Daily News), which was the only local newspaper until 1966.', ' He enjoyed success running many publications before being appointed Governor by President Dwight D. Eisenhower in 1960.', ' As Governor, Flores pushed for increased self-governance in Guam, resigning in 1961.', ' After his Governorship, Flores founded other businesses and became involved in numerous community organizations.', ' He was a Knight of St. Sylvester.']], ['List of current Indian governors', ['In the Republic of India, a governor is the constitutional head of each of the twenty-nine states.', \" The governor is appointed by the President of India for a term of five years, and holds office at the President's pleasure.\", ' The governor is \"de jure\" head of the state government; all its executive actions are taken in the governor\\'s name.', ' However, the governor must act on the advice of the popularly elected council of ministers, headed by the chief minister, who thus hold \"de facto\" executive authority at the state-level.', \" The Constitution of India also empowers the governor to act upon his or her own discretion, such as the ability to appoint or dismiss a ministry, recommend President's rule, or reserve bills for the President's assent.\", ' Over the years, the exercise of these discretionary powers have given rise to conflict between the elected chief minister and the central government–appointed governor.']], ['Jean-Michel de Lepinay', ['Jean-Michel de Lepinay was the governor of the French colony of Louisiana (New France) from 1717 to 1718.', ' Before serving as governor, Lepinay had been a naval officer and served over twenty years in Canada.', ' He was appointed governor by Antoine Crozat, the royally appointed administrator of the colony.', ' Lepinay came to the colony with a new Commissary-Commissioner, fifty new colonists, and three companies of infantry.', ' However, Lepinay soon found himself at odds with Jean-Baptiste Le Moyne de Bienville, twice governor and an influential leader in the colony.', ' Failing to show any improvement over the administration of the former governor, Sieur de Cadillac, Lepinay was replaced as governor by Bienville after Crozat successfully petitioned to be released from his agreement to develop the colony.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.578\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac24cea55429951e9e6853a', 'answer': 'Kate Millett', 'question': 'Which Oxford University graduate did Midge Mackenzie interview in \"Women Talking\"?', 'supporting_facts': [['Midge Mackenzie', 0], ['Kate Millett', 1]], 'context': [['Thum Ping Tjin', ['Thum Ping Tjin (born 17 December 1979), better known as PJ, is a Research Associate at the Centre for Global History and co-ordinator of Project Southeast Asia, University of Oxford.', ' He is the first Singaporean to swim the English Channel.', ' He is also the first Oxford University graduate student to do so.', ' He was a member of the Singapore national swimming team and has represented Singapore at every level, including the 1996 Summer Olympics in Atlanta.', ' He is a Rhodes Scholar and Commonwealth Scholar.']], ['Oxford University Liberal Democrats', ['Oxford University Liberal Democrats (previously Oxford University Liberal Club and Oxford University Social Democrats) is the student branch of the Liberal Democrats for students at the University of Oxford.', ' It is the official successor to both the Oxford University Liberal Club and the Oxford University Social Democrats, which voted to merge early in 1987, about a year in advance of the national parties.']], ['Oxford University Music Society', ['The Oxford University Music Society (OUMS) is one of the oldest societies in the University of Oxford, England, tracing its origins back to 1872.', ' The Society was formed in 1916 by the merger of the Oxford University Musical Club, founded in 1872, and the Oxford University Musical Union, founded in 1884.', ' Originally called the Oxford University Musical Club and Union, it changed its name to the Oxford University Musical Society in 1983.']], ['Women Talking Dirty', ['Women Talking Dirty is a 1999 Scottish comedy film starring Helena Bonham Carter and Gina McKee.', ' It is an adaptation of the novel \"Women Talking Dirty\", written by Isla Dewar who wrote the screenplay as well.', ' The film was screened at the Toronto International Film Festival on 17 September 1999 and released on 7 December 2001 in the UK.']], ['Shoulder to Shoulder', [\"Shoulder to Shoulder is a 1974 BBC television serial and book relating the history of the women's suffrage movement, both edited by Midge Mackenzie.\", ' The drama series grew out of discussions between Mackenzie and the actress and singer Georgia Brown, who was dissatisfied at the lack of decent roles for women in TV drama.', \" Brown enlisted the producer Verity Lambert in the project she and Mackenzie were devising to dramatise the struggle for women's suffrage, and the three women presented the idea to the BBC, which gave approval for the series.\", ' Originally they had hoped to use only female script writers but this proved impracticable.', ' Male writers were used and the three female originators of the project found they needed to remove from their scripts a number of \\'innuendoes, misconceptions and untruths\\' indicative of what Georgia Brown termed \"the male point of view\".']], ['Kate Millett', ['Katherine Murray Millett (September 14, 1934 – September 6, 2017) was an American feminist writer, educator, artist, and activist.', \" She attended Oxford University and was the first American woman to be awarded a degree with first-class honors after studying at St Hilda's College, Oxford.\", ' She has been described as \"a seminal influence on second-wave feminism\", and is best known for her book \"Sexual Politics\" (1970), which was based on her doctoral dissertation at Columbia University.', ' Journalist Liza Featherstone attributes previously unimaginable \"legal abortion, greater professional equality between the sexes, and a sexual freedom\" being made possible partially due to Millett\\'s efforts.']], ['Midge Mackenzie', ['Margaret Rose MacKenzie, known as Midge Mackenzie, (6 March 1938 - 28 January 2004) was a London-born writer and filmmaker who first become known for producing Robert Joffrey\\'s multimedia ballet \"Astarte\" with the Joffrey Ballet, and \"Women Talking\", a documentary with interviews of Kate Millett, Betty Friedan and other leading figures in the US women’s liberation movement.']], ['Oxford University Rowing Clubs', [\"Oxford University Rowing Clubs (OURCs) is a federation of the Oxford University Boat Club (OUBC), the Oxford University Women's Boat Club (OUWBC), the Oxford University Lightweight Rowing Club (OULRC), and the Oxford University Women's Lightweight Rowing Club (OUWLRC), as well as all college boat clubs.\", ' OURCs is a purely administrative organisation with no training or crews.', ' It was created in 1986 in order to remove the organisational burden from the university squad and is responsible for organising inter-collegiate competitions and overseeing the conduct of college rowing.', ' The student-led organisation of OURCs is supported by senior members of the university, the Council for Oxford University Rowing, which issues advice and deals with aspects of rowing safety.']], ['Astarte (ballet)', ['Astarte, choreographed by Robert Joffrey, was the first live, multi-media ballet with a specially commissioned rock music score composed and performed by Crome Syrcus.', ' It received its world premier on September 20, 1967 and was performed by the Joffrey Ballet in New York City at the City Center Theater.', ' It was produced by Midge Mackenzie, with sets and lighting design by Thomas Skelton, costumes by Hugh Sherrer, and film created and photographed by Gardner Compton.']], ['Caryn Davies', ['Caryn Davies (born April 14, 1982 in Ithaca, New York) is an American rower.', \" She won gold medals as the stroke seat in women's eight at the 2012 Summer Olympics and the 2008 Summer Olympics.\", \" In April 2015 Davies stroked Oxford University to victory in the first ever women's Oxford/Cambridge boat race held on the same stretch of the river Thames in London where the men's Oxford/Cambridge race has been held since 1829.\", \" She was the most highly decorated Olympian to take part in either [men's or women's] race.\", ' In 2012 Davies was ranked number 4 in the world by the International Rowing Federation.', \" At the 2004 Olympic Games she won a silver medal in the women's eight.\", ' Davies has won more Olympic medals than any other U.S. oarswoman.', \" The 2008 U.S. women's eight, of which she was a part, was named FISA (International Rowing Federation) crew of the year.\", ' Davies is from Ithaca, New York, where she graduated from Ithaca High School, and rowed with the Cascadilla Boat Club.', \" Davies was on the Radcliffe College (Harvard) Crew Team and was a member on Radcliffe's 2003 NCAA champion Varsity 8, and overall team champion.\", \" In 2013, she was a visiting student at Pembroke College, Oxford, where she stroked the college men's eight to a victory in both Torpids (spring intercollegiate races) and the Oxford University Summer Eights races (for the first time in Oxford rowing history).\", ' In 2013–14 Davies took up Polynesian outrigger canoeing in Hawaii, winning the State novice championship and placing 4th in the long distance race na-wahine-o-ke-kai with her team from the Outrigger Canoe Club.', ' In 2013, she was inducted into the New York Athletic Club Hall of Fame.', \" She has served as a Vice President of the U.S. Olympians Association and as athletes' representative to the Board of USRowing.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.579\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8ec02e5542995a26add501', 'answer': 'Check It Out!', 'question': 'Which American sketch comedy television series starring John C. Reilly was produced by Abso Lutely Productions?', 'supporting_facts': [['Abso Lutely Productions', 1], ['Abso Lutely Productions', 2], ['Abso Lutely Productions', 3], ['Check It Out! with Dr. Steve Brule', 0], ['Check It Out! with Dr. Steve Brule', 1]], 'context': [['Abso Lutely Productions', ['Abso Lutely Productions is a film and television production company owned by actor and producers Tim Heidecker, Eric Wareheim and producer Dave Kneebone.', ' It is known for producing the long-running series \"Tom Goes to the Mayor\", \"Nathan For You\", \"Tim and Eric Awesome Show, Great Job!', '\" and \"Check It Out!', ' with Dr. Steve Brule\".']], ['Kate Berlant', ['Kate Berlant (born July 16, 1987) is an American stand-up comedian, actress, and writer.', ' She is known for appearing on \"The Meltdown with Jonah and Kumail\" and starring in an episode of the Netflix original series \"\".', ' With John Early, Berlant created the Vimeo original series \"555\" produced by Abso Lutely Productions.', ' Her father is the artist Tony Berlant.']], ['Friends of the People', ['Friends of the People is an American sketch comedy television series.', \" It was slated to premiere on TruTV in summer 2014, but was pushed to October 28, 2014, as part of the network's shift in their programming direction.\", ' Many of the cast members (Jennifer Bartels, Jermaine Fowler, and Lil Rel Howery) were originally reported to be cast members of a planned revival of In Living Color which never materialized.', \" The show's first season consists of 10 episodes.\", \" This makes it the network's first sketch comedy show.\", ' The series holds a TV-14 rating, though select episodes are rated TV-MA--also a first for the truTV network.']], ['In Living Color', ['In Living Color is an American sketch comedy television series that originally ran on Fox from April 15, 1990, to May 19, 1994.', ' Brothers Keenen and Damon Wayans created, wrote and starred in the program.', ' The show was produced by Ivory Way Productions in association with 20th Century Fox Television and was taped at stage 7 at the Fox Television Center on Sunset Boulevard in Hollywood, California.', ' The title of the series was inspired by the NBC announcement of broadcasts being presented \"in living color\" during the 1960s, prior to mainstream color television.', ' It also refers to the fact that most of the show\\'s cast were black, unlike other sketch comedy shows such as \"Saturday Night Live\" whose casts are mostly white.', ' It was controversial due to the Wayans\\' decision to portray African-American humor from the ghetto in a time when mainstream American tastes regarding black comedy had been set by more upscale shows such as \"The Cosby Show\", causing an eventual feud for control between Fox executives and the Wayans.']], ['List of Key &amp; Peele episodes', ['\"Key & Peele\" is an American sketch comedy television series starring Keegan-Michael Key and Jordan Peele, both former cast members of \"MADtv\".', ' Each episode of the series consists of several pre-taped sketches starring the two actors.', ' The sketches cover a variety of societal topics, often with a focus on African-American culture and race relations.', ' The series premiered on January 31, 2012 and ended on September 9, 2015, with a total of 53 episodes, over the course of five seasons.', ' A special entitled \"Key & Peele\\'s Super Bowl Special\" aired on January 30, 2015.']], ['Bagboy (TV special)', ['Bagboy is a 2015 American television special produced for Adult Swim, and aired on February 21, 2015, to positive critical reception.', ' Written and directed by Tim Heidecker and Eric Wareheim for Abso Lutely Productions and John C. Reilly, the special is a fictional sitcom pilot in the universe of \"Tim and Eric Awesome Show, Great Job!', '\", written by and starring Reilly\\'s recurring \"Tim and Eric\" character Dr. Steve Brule.', ' This is the second \"Tim and Eric\" spin-off to feature the Dr. Steve Brule character after \"Check it Out!', ' with Dr. Steve Brule\".']], ['Dave Kneebone', ['Dave Kneebone is an American producer.', ' Along with Eric Wareheim and Tim Heidecker he runs Abso Lutely Productions which produces a variety of television and films.', ' His role has been described as the business chief and \"straight man\" at Abso Lutely.', ' Kneebone has worked as a producer on a variety of television shows including \"Comedy Bang!', ' Bang!', '\", \"Nathan for You\", and \"Tim and Eric Awesome Show, Great Job!', '\".', ' He is also listed a producer on the feature film \"Tim and Eric\\'s Billion Dollar Movie\".']], ['Hot Package', ['Hot Package is an Adult Swim entertainment variety show, created by Derrick Beckles.', ' The show parodies network entertainment shows such as \"Entertainment Tonight\" and \"Access Hollywood\".', ' Instead of sourcing its news from real celebrities, TV shows, and films, all of Hot Package\\'s \"entertainment\" news comes from found footage, including clips from forgotten B Films and bizarre TV shows.', \" The show is hosted by Derrick Beckles, Pat O'Brien, Anastasia Roark, and Mark McGrath, and features colorful guests, makeovers, and interview segments.\", ' \"Hot Package\", produced by Abso Lutely Productions, Abominable Pictures, TV Carnage, and Williams Street, premiered on October 4, 2013, and has currently aired eleven episodes.', ' On May 9, 2014, Adult Swim confirmed that Hot Package would be returning for a second season.']], ['Check It Out! with Dr. Steve Brule', ['Check It Out!', ' with Dr. Steve Brule is an American sketch comedy television series that is a spin-off of \"Tim and Eric Awesome Show, Great Job!', '\" starring John C. Reilly as Dr. Steve Brule.', \" The series premiered on Cartoon Network's late night programming block, Adult Swim, on May 16, 2010.\", ' The program follows Brule as he examines different facets of living.', \" His severe naivete and social awkwardness generally land him in embarrassing situations, though he largely remains ignorant of any embarrassment he's causing himself.\", ' As the series progresses, he reveals shocking and sometimes horrifying details about his past and personal life.']], ['Upright Citizens Brigade (TV series)', ['Upright Citizens Brigade is an American sketch comedy television series that premiered on August 19, 1998 on Comedy Central.', ' The show aired for three seasons with each season consisting of ten episodes.', ' The series featured four members of Upright Citizens Brigade, an improvisational sketch comedy group.', ' The cast included Matt Besser, Amy Poehler, Ian Roberts, and Matt Walsh.', ' The cast would later reunite for another series of a similar format that premiered in 2016 on Seeso.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.580\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac1276c554299294b2190bc', 'answer': 'Euthanasia Program', 'question': \"Heinrich Gross participated in the Nazi regime's Aktion T4 more commonly referred to as what?\", 'supporting_facts': [['Heinrich Gross', 0], ['Aktion T4', 3]], 'context': [['Karl Freiherr Michel von Tüßling', ['Karl Freiherr Michel von Tüßling (27 July 1907 – 30 October 1991) was a Schutzstaffel (SS) officer who served in the Nazi government of German dictator Adolf Hitler and in the SS Main Office.', ' From 1936 onwards, he was the personal adjutant of \"Reichsleiter\" and SS-\"Obergruppenführer\" Philipp Bouhler, who was in charge of Hitler\\'s Chancellery (Kanzlei des Führers), head of the euthanasia programme Aktion T4, as well as co-initiator of Aktion 14f13.', ' In 1947 Tüßling provided an affidavit in defence of war criminal Viktor Brack who was sentenced to death at the Nuremberg trials.']], ['Heinrich Gross', ['Heinrich Gross (14 November 1915 – 15 December 2005) was an Austrian psychiatrist, medical doctor and neurologist, a reputed expert as a leading court-appointed psychiatrist, ill-famed for his proven involvement in the killing of at least nine children with physical, mental and/or emotional/behavioral characteristics considered \"unclean\" by the Nazi regime, under its Euthanasia Program.', ' His role in hundreds of other cases of infanticide is unclear.', \" Gross was head of the Spiegelgrund children's psychiatric clinic for two years during World War II.\"]], ['Am Spiegelgrund clinic', [\"Am Spiegelgrund was the name of a children's clinic in Vienna where 789 children were killed under the Nazi Regime Children's Euthanasia Program, also known as Aktion T4.\", ' Between 1940-1945, the clinic operated as part of the psychological institution “Am Steinhof” (renamed the Otto Wagner Clinic) on the Baumgartner Höhe, now located in Penzing, the 14th district of Vienna.', ' This clinic was divided into a reform school and a sanatorium for children, which included a so-called Children’s Ward, where sick, disabled, and otherwise ‘un-educable’ adolescents were abused and subjected to harsh medical experiments.', \" Some died by lethal injection and gas poisoning; others by disease, undernourishment, exposure to the elements, and 'accidents' relating to their conditions.\", ' The brains of up to 800 victims were preserved in jars and housed in the hospital for decades.']], ['Philipp Bouhler', ['Philipp Bouhler (11 September 1899 – 19 May 1945) was a senior Nazi Party official who was both a \"Reichsleiter\" (National Leader) and Chief of the Chancellery of the Führer of the NSDAP.', ' He was also an SS-\"Obergruppenführer\" in the \"Allgemeine SS\" who was responsible for the Nazi \"Aktion T4\" euthanasia program that killed more than 70,000 handicapped adults and children in Nazi Germany, as well as co-initiator of \"Aktion 14f13\", also called \"Sonderbehandlung\" (\"special treatment\"), that killed 15,000–20,000 concentration camp prisoners.']], ['Artur Hojan', ['Artur Hojan (7 August 1973 – found dead, 9 February 2014) was a journalist and published author specializing in the history of the Chełmno extermination camp and the Nazi involuntary euthanasia programme conducted in the territory of occupied Poland by the SS during World War\\xa0II.', ' Hojan was the co-founder of the \"Tiergartenstrasse4\" Association in 2005 (together with Cameron Munro) devoted to Aktion T4 history, with emphasis on the Kościan psychiatric hospital located where he lived.', ' Hojan, age of 40, left home in the evening of 1\\xa0December 2013 at 8\\xa0p.m. for a walk around town and disappeared.', ' His body was found two months later on 9\\xa0February 2014 floating in the Obra canal near the town of Kiełczewo, and identified later.', ' The cause of death has not been determined.', ' He was buried at the Kościan cemetery on 15\\xa0February 2014.', ' He left behind a wife and young daughter.', ' The monograph \"Treblinka Death Camp: History, Biographies, Remembrance\" by Chris Webb, the co-founder of H.E.A.R.T (also known as the HolocaustResearchProject.org), is dedicated to his memory.']], ['Aktion T4', ['Aktion T4 (German, ] ) was a postwar name for mass murder through involuntary euthanasia in Nazi Germany.', ' The name T4 is an abbreviation of \"Tiergartenstraße 4\", a street address of the Chancellery department set up in the spring of 1940, in the Berlin borough of Tiergarten, which recruited and paid personnel associated with T4.', ' Certain German physicians were authorized to select patients \"deemed incurably sick, after most critical medical examination\" and then administer to them a \"mercy death\" (\"Gnadentod\") .', ' In October 1939 Adolf Hitler signed a \"euthanasia decree\" backdated to 1 September 1939 that authorized his personal physician Karl Brandt and \"Reichsleiter\" Philipp Bouhler to implement the programme.']], ['Karl Brandt', ['Karl Brandt (January 8, 1904 – June 2, 1948) was a German physician and \"Schutzstaffel\" (SS) officer in Nazi Germany.', \" Trained in surgery, Brandt joined the Nazi Party in 1932 and became Adolf Hitler's escort physician in August 1934.\", ' A member of Hitler\\'s inner circle at the Berghof, he was selected by Philipp Bouhler, the head of Hitler\\'s Chancellery, to administer the \"Aktion T4\" euthanasia program.', ' Brandt was later appointed the Reich Commissioner of Sanitation and Health (\"Bevollmächtigter für das Sanitäts- und Gesundheitswesen\").', ' Accused of involvement in human experimentation and other war crimes, Brandt was indicted in late 1946 and faced trial before a U.S. military tribunal along with 22 others in \"United States of America v. Karl Brandt, et al\".', ' He was convicted, sentenced to death, and later hanged on June 2, 1948.']], ['Gerhard Kretschmar', ['Gerhard Herbert Kretschmar (20 February 1939 – 25 July 1939), was a German child born with severe disabilities.', \" After receiving a petition from the child's parents, the German Führer Adolf Hitler authorized one of his personal physicians, Karl Brandt, to have the child killed.\", ' This marked the beginning of the program in Nazi Germany known as a \"euthanasia program\" (Aktion T4) which ultimately resulted in the deliberate killing of about 200,000 people with mental and/or physical disabilities.']], ['Memorandum Authorizing Involuntary Euthanasia', ['Adolf Hitler signed a memorandum authorizing involuntary euthanasia in October 1939 to serve as the legal basis for Aktion T4, the Nazi forced euthanasia program.', ' Its purpose was to assure the doctors and nurses who took part in the euthanasia program would not be prosecuted for murder.', ' During the postwar trials of these same individuals, they attempted to use this decree as a justification for their actions.']], ['Dasein ohne Leben', ['Dasein ohne Leben – Psychiatrie und Menschlichkeit (\"Existence Without Life\" – \"Psychiatry and Humanity\") is a 1942 Nazi propaganda film about the physically and mentally disabled: closeups of disabled persons.', ' The director was Hermann Schwenninger, one of the three managing directors of Gemeinnützige Krankentransport (\"Charitable Ambulance\"), a front company of Aktion T4, the central institution for the mass murder of patients in the Third Reich.', ' Schwenninger also wrote parts of the screenplay of \"Ich klage an\".', \" The contract for the film came from Hitler's Chancellery, and was produced by Tobis Film.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.580\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7280105542994cef4bc2e4', 'answer': 'Limbo', 'question': 'Indie studio Jumpship developed what puzzle-platform video game released in July 2010?', 'supporting_facts': [['Somerville (video game)', 0], ['Limbo (video game)', 0], ['Limbo (video game)', 1]], 'context': [['Limbo (video game)', ['Limbo is a puzzle-platform video game developed by independent studio Playdead.', ' The game was released in July 2010 on Xbox Live Arcade, and has since been ported to several other systems, including the PlayStation 3 and Microsoft Windows.', ' \"Limbo\" is a 2D side-scroller, incorporating a physics system that governs environmental objects and the player character.', ' The player guides an unnamed boy through dangerous environments and traps as he searches for his sister.', \" The developer built the game's puzzles expecting the player to fail before finding the correct solution.\", ' Playdead called the style of play \"trial and death\", and used gruesome imagery for the boy\\'s deaths to steer the player from unworkable solutions.']], ['Antichamber', ['Antichamber is a single-player first-person puzzle-platform video game created by Alexander Bruce.', ' Many of the puzzles are based on phenomena that occur within impossible objects created by the game engine, such as passages that lead the player to different locations depending on which way they face, and structures that seem otherwise impossible within normal three-dimensional space.', ' The game includes elements of psychological exploration through brief messages of advice to help the player figure out solutions to the puzzles as well as adages for real life.', ' The game was released on Steam for Microsoft Windows on January 31, 2013, a version sold with the Humble Indie Bundle 11 in February 2014 added support for Linux and Mac OS X.']], ['Metrico', ['Metrico is an indie puzzle-platform video game developed and published by Dutch developer Digital Dreams for the PlayStation Vita, with music by Dutch electronic music producer Palmbomen.', ' It was released in North America on 5 August 2014, and in Europe on 6 August 2014.', ' \"Metrico\" was initially available for free for members of PlayStation Plus.']], ['Portal 2', ['Portal 2 is a 2011 first-person puzzle-platform video game developed and published by Valve Corporation.', ' It is the sequel to \"Portal\" (2007) and was released on April 19, 2011, for Microsoft Windows, OS X, Linux, PlayStation 3, and Xbox 360.', \" The retail versions of the game are distributed by Electronic Arts while online distribution of the Microsoft Windows, Mac OS X and Linux versions is handled by Valve's content delivery service Steam.\", ' \"Portal 2\" was announced on March 5, 2010, following a week-long alternate reality game based on new patches to the original game.', ' Before the game\\'s release on Steam, the company released the Potato Sack, a second multi-week alternate reality game, involving 13 independently developed titles which culminated in a distributed computing spoof to release \"Portal 2\" several hours early.']], ['Teslagrad', ['Teslagrad is a 2D side-scrolling puzzle-platform video game developed and published by Rain Games.', ' The game was released on Steam on 13 December 2013, on Nintendo eShop for the Wii U on 11 September 2014, and on PlayStation Store for the PlayStation 3 and PlayStation 4 on 3 December 2014 in Europe.', ' The North American PSN Store release date for the PlayStation 3 and PlayStation 4 was scheduled on 27 January 2015, but it has been postponed on the day of the release.', ' The retail version, published by Soedesco, was released in France, Germany, Italy, and Spain on 3 December 2014, and in UK on 30 January 2015.', ' The PlayStation Vita version was announced on 27 February 2014, but since then no exact date was given.', ' The game was also released digitally in Japan on 18 February 2015, with an update to the Steam version that added Japanese language.', ' An Xbox One version was announced at the end of February 2016 with the release date of 9 March 2016.', ' \"Teslagrad\" uses Unity game engine, making it easier for developers to reach cross platform.']], ['The Lost Vikings', ['The Lost Vikings is a puzzle-platform video game developed by Silicon & Synapse (now Blizzard Entertainment) and published by Interplay.', ' It was originally released for the Super NES in 1992, then subsequently released for the Amiga, Amiga CD32, MS-DOS, and Mega Drive/Genesis systems the next year; the Mega Drive/Genesis version contains five stages not present in any other version of the game.', ' Blizzard re-released the game for the Game Boy Advance in 2003.', ' In 2014, the game was added to Battle.net as a free download emulated through DOSBox.']], ['Fez (video game)', ['Fez (stylized as FEZ) is an indie puzzle-platform video game developed by Polytron Corporation and published by Trapdoor.', ' The player-character Gomez receives a fez that reveals his two-dimensional (2D) world to be one of four sides of a three-dimensional (3D) world.', \" The player rotates between these four 2D views to realign platforms and solve the game's puzzles.\", ' The object of the game is to collect cubes and cube fragments to restore order to the universe.']], ['Pitman (video game)', ['Pitman, also known as Catrap in the US, is a puzzle-platform video game released by Asmik for the Nintendo Game Boy in 1990, originally developed for the Sharp MZ-700 computer in 1985.', ' The Game Boy version of Pitman was rereleased on the Nintendo 3DS Virtual Console in October 2011.', \" The word 'Catrap' refers to the frequent amount of times the player is trapped and needs to reverse their movements and the two anthropomorphic cats the player must manoeuvre to advance through the levels.\", ' The game is credited with having originated the time-rewind mechanic, which later appeared in titles like \"\", \"\", \"Braid\" and Pullblox.']], ['Black the Fall', ['Black the Fall is an indie puzzle-platform video game developed by Sand Sailor Studio and published by Square Enix for Linux, Microsoft Windows, OS X, PlayStation 4, and Xbox One.']], ['Somerville (video game)', ['Somerville is an upcoming video game and the debut title by the indie studio Jumpship.', ' The studio\\'s founder previously founded Playdead and worked on \"Limbo\" and \"Inside\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.581\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae6e0485542996d980e7ca6', 'answer': 'Mani', 'question': 'Who was the winner of the season of The Rap Game where a contestant named Lady Leshurr as one of her influences?', 'supporting_facts': [['Lady Leshurr', 2], ['The Rap Game', 2]], 'context': [['Lady and the Tramp', ['Lady and the Tramp is a 1955 American animated romantic musical comedy-drama film produced by Walt Disney and released to theaters on June 22, 1955 by Buena Vista Distribution.', ' The 15th Disney animated feature film, it was the first animated feature filmed in the CinemaScope widescreen film process.', ' Based on \"Happy Dan, The Whistling Dog\" by Ward Greene, \"Lady and the Tramp\" tells the story of a female American Cocker Spaniel named Lady who lives with a refined, upper-middle-class family, and a male stray mongrel called the Tramp.', ' When the two dogs meet, they embark on many romantic adventures.', ' A direct-to-video sequel, \"\", was released in 2001.']], ['Suran (singer)', ['Shin Su-ran (Hangul: 신수란), also known by her stage names Elena (Hangul: 엘에나) and Baily Shoo (Hangul: 베일리 슈), and better known by the mononym Suran (Hangul: 수란), is a South Korean singer-songwriter and record producer.', ' She debuted as part of the duo, Lodia, on July 9, 2014, with the single \"I Got A Feeling\".', ' In 2017, she has also appeared on MBC\\'s \"King of Mask Singer\" (episodes 93–94) as a contestant named \"Skip to the End, Hello\".']], ['Wonky (album)', ['Wonky is the eight studio album by Orbital, released on their own ACP label (via Warner Music Group/Alternative Distribution Alliance) in the UK on 2 April 2012, and exclusively through iTunes in the USA and Canada on 17 April 2012.', ' The album is their first since the \"Blue Album\" in 2004 and the first since they reformed in 2008.', ' It features vocals from Zola Jesus and Lady Leshurr.', ' The album was taken off of Spotify and iTunes in the United States for unknown reasons.', ' There are some songs you cannot find at all in their original versions, like Beelzedub or Distractions.']], ['Lady Leshurr', [\"Melesha O'Garro (born 15 December 1988), known professionally as Lady Leshurr ( ), is an English rapper, singer, and producer.\", ' She is best known for her \"Queen\\'s Speech\" series of freestyles, the fourth of which went viral in 2016. \"', 'The Rap Game\" season 2 competitor Nia Kay stated that Leshurr was one of her influences due to her uniqueness and rap skill.', ' Her subsequent freestyle, \"Queen\\'s Speech 5\", was called \"brilliant\" and \"2015\\'s crowning freestyle\" by Spin.']], ['Donkey Kong (video game)', ['Donkey Kong (Japanese: ドンキーコング , Hepburn: Donkī Kongu ) is an arcade game released by Nintendo in 1981.', ' An early example of the platform game genre, the gameplay focuses on maneuvering the main character across a series of platforms while dodging and jumping over obstacles.', ' In the game, Mario (originally named Mr. Video and then Jumpman) must rescue a damsel in distress named Pauline (originally named Lady), from a giant ape named Donkey Kong.', \" The hero and ape later became two of Nintendo's most popular and recognizable characters.\", ' \"Donkey Kong\" is one of the most important titles from the golden age of arcade video games, and is one of the most popular arcade games of all time.']], ['Nuestra Belleza Latina 2009', ['Nuestra Belleza Latina 2009 is the third season of Nuestra Belleza Latina (Our Latin Beauty) premiered on March 2009.', ' Auditions were once again held in five major US cities (Los Angeles, California; Dallas, Texas; Miami, Florida; Chicago, Illinois; and New York City, New York) and in San Juan, Puerto Rico.', ' During the audition process, 75 young women were given passes to the semi-finals in Miami, Florida.', ' With the twist of adding one more contestant named \"La Intrusa\" (Francheska Mattei).', ' For several weeks, Francheska Mattei, a professional actress, posed as one of the contestants to know the rumors, secrets, and even gossip from the girls.', ' The elimination process was quick, with 15 women leaving the first day, 40 on the second week, and finally 8 women were eliminated leaving the 12 finalists who will be moving into a Miami mansion where they will be living together for the rest of the competition.', ' Each week, viewers will have a chance to vote for their favorite finalists.', ' The three women with the least number of votes will then be in danger of being eliminated.', ' However, two of the women in the bottom three will have the chance of being saved, one by her fellow finalists, and the other by the judges.', ' This year they had an \"Intrusa\", the one that spilled the gossip about the contestants and it was Puerto Rican, Francheska Mattei.']], ['The Rap Game', ['The Rap Game is an American reality television series.', ' The series premiered on January 1, 2016, on Lifetime.', ' The winners of seasons 1, 2, and 3, respectively, were Miss Mulatto, Mani, and Nova.']], ['Nick Donnelly', ['Nicholas James Donnelly (born 17 May 1988) is a British filmmaker and music video producer who first gained exposure when directing/producing the music video Game Over Female Takeover, an independent release that featured many of the leading female urban artists in England on one video.', ' These artists included Lady leshurr, Mz Bratt, Ruff Diamondz, Cherri Voncelle and Amplify Dot The video served as the official remix to the record Game Over (Tinchy Stryder song)']], ['Expeditie Robinson 2006', ['Expeditie Robinson: 2006, was the eighth Dutch/Belgian version of the Swedish show Expedition Robinson, or Survivor as it is referred to in some countries.', ' This season began airing on August 28, 2006 and concluded on November 20, 2006.', ' The major twist this season was that the tribes were initially divided up by gender with one \"All-Star\" contestant joining each tribe.', ' These two All-Stars were, Klaar Lippe and Robin Ibens.', ' Though the main twist may have seemed like a repeat of twists that occurred in previous seasons, unlike previous seasons with a similar twist this season male contestants were from Belgium and all females from the Netherlands.', \" While the All-Star contestants weren't eligible to win, they could vote and following a tribal swap in episode 4, they could give any contestant on their tribe immunity at tribal council.\", ' Following the merge in episode 8, the twist that was \"Losers Island\", was introduced to the game.', ' After a contestant was eliminated they would be sent to Losers island where they would wait until there were only two contestants left in the game at which point the six contestants on the island would vote for one of their own to return.', ' When it came time to vote for a winner, the public, instead of a jury decided the winner.', \" In the end, it was Olga Urashova who returned from Losers island in the final three, who won the season over Lenny Janssen with a public vote of 52% to Lenny's 48%.\"]], ['Bell Nuntita', ['Nuntita Khampiranon (Thai: นันทิตา ฆัมภิรานนท์ ; rtgs:\\xa0\"Nanthita Khamphiranon\" ; born December 20, 1983), or nickname Art (อาร์ต), stage name Bell (เบลล์) and usually known as Bell Nuntita, is a Thai transgender actress, singer, entertainer, and radio DJ.', ' Nuntita was part of a TV show called \"Venus Flytrap Search for the Missing Puzzle\" in 2007.', ' As one of twelve contestants, she competed to take the place of two former cast members.', ' Nuntita and another contestant named Mew won the competition and became part of the Kathoey band \"Venus Flytrap\".', ' She became popular after a performance during her audition on \"Thailand\\'s Got Talent\" and became a YouTube hit when she first performed singing as a girl, and the crowd was amazed when she switched to a masculine voice.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.582\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae031c455429924de1b7055', 'answer': 'six', 'question': 'How many major novels did the author write, whose 1811 book was adapted for a 2008 film with a screenplay by Andrew Davies ?', 'supporting_facts': [['Sense and Sensibility (2008 miniseries)', 0], ['Sense and Sensibility (2008 miniseries)', 1], ['Jane Austen', 0]], 'context': [['Northanger Abbey (2007 film)', [\"Northanger Abbey is a 2007 British television film adaptation of Jane Austen's eponymous novel.\", ' It was directed by British television director Jon Jones and the screenplay was written by Andrew Davies.', ' Felicity Jones stars as the protagonist Catherine Morland and JJ Feild plays her love interest Henry Tilney.', ' The story unfolds as the teenaged Catherine is invited to Bath to accompany some family friends.', \" There she finds herself the object of Henry Tilney's and John Thorpe's (William Beck) affections.\", \" When she is asked to stay at Northanger Abbey, Catherine's youthful and naive imagination takes hold and she begins to confuse real life with the Gothic romance of her favorite novels.\"]], ['Jane Austen', ['Jane Austen ( ; 16 December 1775 – 18 July 1817) was an English novelist known primarily for her six major novels, which interpret, critique and comment upon the British landed gentry at the end of the 18th century.', \" Austen's plots often explore the dependence of women on marriage in the pursuit of favourable social standing and economic security.\", ' Her works critique the novels of sensibility of the second half of the 18th century and are part of the transition to 19th-century literary realism.']], ['Diana (TV series)', ['Diana is a British television drama series first broadcast by the BBC in 1984.', ' It was adapted by Andrew Davies from two R. F. Delderfield novels.']], ['Quirke (TV series)', ['Quirke is a British-Irish crime drama television series that was first broadcast on BBC One and RTÉ One in 2014.', ' The three-part series is based on the Quirke novels by John Banville, writing under the pseudonym Benjamin Black, and was adapted by Andrew Davies and Conor McPherson.']], ['Alfonso Bonzo', [\"Alfonso Bonzo is a 1986 children's book by Andrew Davies and a 1990 children's television mini-series adapted from the book by the author.\", ' The series starred Alex Jennings as Alfonso Bonzo and Scott Riley as Billy Webb.']], ['Brideshead Revisited (film)', ['Brideshead Revisited is a 2008 British drama film directed by Julian Jarrold.', ' The screenplay by Jeremy Brock and Andrew Davies is based on the 1945 novel of the same name by Evelyn Waugh, which previously had been adapted in 1981 as an the television serial \"Brideshead Revisited\".']], ['He Knew He Was Right', ['He Knew He Was Right is an 1869 novel written by Anthony Trollope which describes the failure of a marriage caused by the unreasonable jealousy of a husband exacerbated by the stubbornness of a wilful wife.', \" As is common with Trollope's works, there are also several substantial subplots.\", ' Trollope makes constant allusions to Shakespeare\\'s \"Othello\" throughout the novel.', ' Trollope considered this work to be a failure; he viewed the main character as unsympathetic, and the secondary characters and plots as much more lively and interesting, but it is one of his best known novels.', ' It was adapted for BBC One in 2004 by Andrew Davies as \"He Knew He Was Right\".']], ['Affinity (film)', [\"Affinity is a 2008 UK film adaptation of Sarah Waters' 1999 novel of the same name; directed by Tim Fywell and screenplay by Andrew Davies.\", ' The film was nominated for the GLAAD Media Award for Outstanding TV Movie or Limited Series.']], ['Sense and Sensibility (2008 miniseries)', ['Sense and Sensibility is a 2008 British television drama adaptation of Jane Austen\\'s 1811 novel \"Sense and Sensibility\".', ' The screenplay was written by Andrew Davies, who revealed that the aim of the series was to make viewers forget Ang Lee\\'s 1995 film \"Sense and Sensibility\".', ' The series was \"more overtly sexual\" than previous Austen adaptations, and Davies included scenes featuring a seduction and a duel that were absent from the feature film.', ' \"Sense and Sensibility\" was directed by John Alexander and produced by Anne Pivcevic.', ' Hattie Morahan and Charity Wakefield star as Elinor and Marianne Dashwood, two sisters who go on \"a voyage of burgeoning sexual and romantic discovery\".']], ['John Cowper Powys', ['John Cowper Powys ( ; 8 October 187217 June 1963) was a British philosopher, lecturer, novelist, literary critic, and poet.', ' Although Powys published a collection of poems in 1896 and his first novel in 1915, he did not gain success as a writer until he published the novel \"Wolf Solent\" in 1929.', ' He was influenced by many writers, but he has been particularly seen as a successor to Thomas Hardy, and \"Wolf Solent\", \"A Glastonbury Romance\" (1932), along with \"Weymouth Sands\" (1934) and \"Maiden Castle\" (1936), are often referred to as his Wessex novels.', \" As with Hardy's novels, the landscape plays a major role in Powys's works, and an elemental philosophy is important in the lives of his characters.\", ' In 1934 he published his important \"Autobiography\".', ' Powys was also a highly successful itinerant lecturer, first in England and then from 1905 until 1930 in the USA.', ' Many of Powys\\'s novels were written in America and his early novels, and all his major novels, up to and including \"Owen Glendower\" (1940), as well as \"Autobiography,\" were first published in the United States']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.583\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab262a4554299340b5254ac', 'answer': 'The Most Dangerous Game', 'question': \"What was Richard Connell's best known work The Hounds of Zaroff also published as?\", 'supporting_facts': [['Seven Faces', 1], ['The Most Dangerous Game', 0]], 'context': [['Seven Faces', ['Seven Faces is a 1929 American pre-Code drama film with fantasy elements that was released by Fox Film Corporation in the Fox Movietone sound-on-film system on December 1, 1929.', ' Based upon the piece of short fiction \"A Friend of Napoleon\" which was published in the June 30, 1923, issue of \"The Saturday Evening Post\" magazine by popular writer Richard Connell (whose best known work, \"The Most Dangerous Game\", was filmed three years later), it was directed by Berthold Viertel and stars Paul Muni in his second screen appearance.', ' \"Seven Faces\" is a lost film, with no excerpts from its footage known to exist.']], ['Patrick Burns (paranormal investigator)', ['Patrick Burns (born 1968) is an American paranormal investigator, best known as star of the TruTV (formerly Court TV) series \"Haunting Evidence\".', ' He is the founder of the popular website Ghost Hounds, which in 2001 was featured in an Emmy award-winning Turner documentary \"Interact Atlanta - \\'Ghost Hounds\\'\\xa0\".', ' Burns is also the organizer and director of Ghostock, paranormal enthusiast events held at various locations across the USA, and is a professional photographer through Patrick Burns Photography.']], ['Thrill of a Romance', ['Thrill of a Romance (also known as \"Thrill of a New Romance\") was an American romance film released by Metro-Goldwyn-Mayer in 1945, starring Van Johnson, Esther Williams and Carleton G. Young, with musical performances by opera singer Lauritz Melchior.', ' The film was directed by Richard Thorpe and written by Richard Connell and Gladys Lehman.']], ['The Most Dangerous Game', ['\"The Most Dangerous Game\", also published as \"The Hounds of Zaroff\", is a short story by Richard Connell, first published in \"Collier\\'s\" on January 19, 1924.', ' The story features a big-game hunter from New York City who falls off a yacht and swims to an isolated island in the Caribbean, where he is hunted by a Russian aristocrat.', ' The story is inspired by the big-game hunting safaris in Africa and South America that were particularly fashionable among wealthy Americans in the 1920s.']], ['Cuthy Mede', ['Cuthy Mede is a Malawian artist.', ' Lonely Planet said \"possibly the best-known [Malawian] artist is Cuthy Mede – he is also actively involved in the development and promotion of Malawian art within the country and around the world.\"', ' Cuthy Mede grew up on Likoma Island, Lake Malawi where he drew in the rough sands of the beach as a child.', ' Later he studied Fine Art in Chancellor College and became a lecturer at the College in the 1970s.', ' By the 1980s Mede established Gallerie Africaine in Lilongwe City Centre, the first art gallery by a local artist in Malawi.', ' Mede exhibited his work widely in Malawi, becoming a successful artist selling his work to international collectors.', ' Mede encouraged the work of young Malawian artists struggling to make a living selling folk art and wood carvings as street traders.', ' He also brought fine art work from other Malawian artists into his Gallery.', ' He was commissioned to paint a large mural decorating the City Centre.', ' Mede is best known for his modern art styles: modern, futurist, cubist and pointillist, with strong local themes.', ' His paintings depicted local people, historic events and current events in Malawi, Biblical references with local interpretations, indigenous religious expressions, and paintings about ideas such as Justice, Greed, Man and Machine.', ' His paintings depict famine, refugees from Mozambique during the Civil War, voting and democracy, wedding celebration, spirits and possession, and the Nyau masquerade.', \" Mede's less known work is realistic, including a reproduction of the Mona Lisa.\", ' His best known work is dominated by bright primary colors, cubist style, though his pointillist work favors ochres and softer tones in the overall effect.', ' In later years Mede painted mostly in shades of blue, then white on white, the purest light.', ' Mede is an evangelical Christian and his work begins with a point of light from which the rest of the painting flows, the energy from God.', ' This point of light is evident in most of his paintings as a single dot, a sun or moon, or an orb.', ' Best known for his paintings, Mede also produced sculptural forms such as wood figures covered in beads and pigments.', ' His garden in Lilongwe was made into a work of art, in white and light, with fluorescent light tubes hanging from trees and white painted rocks lining the drive and entry.', \" Mede's wife, Esther (deceased 2009), served as Principal Secretary for the Ministry of Research and Environmental Affairs in the Malawi government.\"]], ['Richard P. Gabriel', ['Richard P. Gabriel (born 1949) is an American computer scientist who is known for his work related to the Lisp programming language (and especially Common Lisp) in computing.', ' His best known work was a 1990 essay “Lisp: Good News, Bad News, How to Win Big”, which incorporated the phrase Worse is Better, and his set of Lisp benchmarks (the \"Gabriel Benchmarks\"), published in 1985 as \"Performance and evaluation of Lisp systems\", which became a standard way of benchmarking Lisp implementations.']], ['E. P. Thompson', ['Edward Palmer Thompson (3 February 1924 – 28 August 1993), usually cited as E. P. Thompson, was a British historian, writer, socialist and peace campaigner.', ' He is probably best known today for his historical work on the British radical movements in the late 18th and early 19th centuries, in particular \"The Making of the English Working Class\" (1963).', ' He also published influential biographies of William Morris (1955) and (posthumously) William Blake (1993) and was a prolific journalist and essayist.', ' He also published the novel \"The Sykaos Papers\" and a collection of poetry.', ' His work is considered to have been among the most important contributions to labour history and social history in the latter twentieth-century, with a global impact, including on scholarship in Asia and Africa.']], ['LRRC (Luddite Rural Recording Cooperative)', [\"LRRC (Luddite Rural Recording Cooperative) was an online music mail-order and CDR/vinyl-focused independent record label run by Indiana lo-fi musician Joseph O'Connell, best known for his band Elephant Micah.\", \" In addition to some of O'Connell's own work, the label released music by Elephant Micah collaborator Jason Henn.\", ' The LRRC online store also stocked a variety of what O\\'Connell called \"Homemade Music from Kentuckiana and Abroad,\" including recordings by regional underground bands like Vollmar, Mt. Gigantic, and Bronze Float.']], ['Andrea Alpago', ['Andrea Alpago (c. 1450 – late 1521 or January 1522) was an Italian physician and arabist.', ' In publications of his work in Latin his name is frequently given as Andreas Alpagus Bellunensis, where \"Bellunensis\" refers to his birthplace of Belluno in northeastern Italy.', ' He worked in Damascus in Syria for decades as physician to the consulate of Republic of Venice in Damascus.', ' He was appointed professor of medicine in Padua in northeastern Italy in 1521, where he taught for only two or three months before his death.', ' None of his works were published during his lifetime; after his death they were published on the initiative of his nephew Paolo Alpago.', ' His best known work is his commentary and editing of the Latin translation of \"The Canon of Medicine\" of Ibn Sina.', ' This medicine book was translated from Arabic to Latin in the late 12th century by Gerard of Cremona.', \" Andrea Alpago's edition and supplements to Gerard of Cremona's translation was widely read in European medical circles during the 16th century.\", ' It was first published in 1527 and an expanded edition was published in 1544.']], ['William Henry Oliphant Smeaton', ['William Henry Oliphant Smeaton (24 October 1856 – 31 March 1914), sometimes using the pen name Oliphant Smeaton, was a Scottish writer, journalist, editor, historian and educator.', \" He was popularly known for his writing on Australian life and literature for various British publications as well as for his adventure and children's fiction novels during the 1890s.\", ' Later in his career, Smeaton also published books on Scottish antiquities and edited English literary text, ballads and collections of verse and prose.', ' His best known work, \"The Life and Works of William Shakespeare\" (1911), was especially successful and enjoyed several reprints.', ' He also contributed several biographies for the \"Famous Scots Series\" published by Oliphant, Anderson and Ferrier.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.583\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a82930655429966c78a6a5e', 'answer': '2010 NCAA Division', 'question': 'Which season did the UCF Knights represent at an American metropolitan public research university in Orlando, Florida?', 'supporting_facts': [['2010 UCF Knights football team', 0], ['University of Central Florida', 0]], 'context': [['2008 UCF Knights football team', ['The 2008 UCF Knights football team represented the University of Central Florida in the 2008 NCAA Division I FBS football season.', \" Their head coach was George O'Leary, in his fifth season with the team.\", \" For the second season, the UCF Knights played all of their home games at Bright House Networks Stadium on the school's main campus in Orlando, Florida.\", ' The Knights sought unsuccessfully to defend their Conference USA football championship.']], ['List of University of Central Florida alumni', ['The University of Central Florida (UCF) is a metropolitan public research and space-grant university located on a 1,415-acre (5.73\\xa0km) main campus in Orlando, Florida, United States.', ' UCF is a member institution of the State University System of Florida and is the largest university in the United States in terms of undergraduate enrollment.', \" It was founded in 1963 as Florida Technological University with the goal of providing highly trained personnel to support the Kennedy Space Center and Cape Canaveral Air Force Station on Florida's Space Coast.\", \" After the university's academic scope expanded in the mid and late 1970s to encompass a wider variety of disciplines, the school was renamed The University of Central Florida in 1978.\", ' Initial enrollment in 1968 was 1,948 students; as of 2014, the university has 59,770 students from more than 140 countries, all 50 U.S. states and the District of Columbia.', \" Since the university's first graduating class in 1970, UCF has awarded more than 250,000 degrees, including 45,000 graduate and professional degrees, to over 200,000 alumni.\"]], ['UCF Knights softball', ['The UCF Knights softball program represents the University of Central Florida in the sport of softball.', ' The Knights compete in Division I of the National Collegiate Athletics Association (NCAA) and the American Athletic Conference (The American).', \" The Knights play their home games at the UCF Softball Complex on UCF's main campus in Orlando, Florida.\", ' The Knights are coached by head coach Renee Luers-Gillispie.', ' In the fourteen-year history of the program, the Knights have won two American regular season championships, three conference tournament championships, and have six appearances in the NCAA Tournament.']], ['University of Central Florida', ['The University of Central Florida, or UCF, is an American metropolitan public research university in Orlando, Florida.', ' It is the largest university in the United States by undergraduate enrollment, as well as the largest enrollment at a single campus.']], ['CFE Arena', ['CFE Arena, officially the CFE Federal Credit Union Arena and formerly known as UCF Arena, is a sports and entertainment arena located in Orlando, Florida, United States on the main campus of the University of Central Florida.', ' It was constructed beginning in 2006 as a replacement for the original UCF arena, and as a part of Knights Plaza.', \" The arena is home to the UCF Knights men's and women's basketball teams.\", ' In 2010, the Legends Football League team Orlando Fantasy played at the arena.', ' The Arena also hosted the annual Science Olympiad in 2012 and 2014.', ' For the 2014 season only, it served as the home of the Orlando Predators of the Arena Football League.']], ['Florida International University', ['Florida International University (FIU) is an American metropolitan public research university in Greater Miami, Florida, United States.', ' FIU has two major campuses in Miami-Dade County, with its main campus in University Park.', ' Florida International University is classified as a research university with highest research activity by the Carnegie Foundation and a research university by the Florida Legislature.']], ['List of UCF Knights football seasons', ['The UCF Knights college football team competes as part of the National Collegiate Athletic Association (NCAA) Division I Football Bowl Subdivision, representing The University of Central Florida in the American Athletic Conference (The American).', \" Since the program's first season in 1979 under Don Jonas, the Knights have played over 400 regular-season games, earning 216 official victories.\", ' UCF and has won four division championships (2005, 2007, 2010, 2012), four conference championships (2007, 2010, 2013, 2014), and has made six postseason appearances since joining FBS (2005, 2007, 2009, 2010, 2012), including the 2014 Fiesta Bowl, a BCS Bowl.', ' The Knights current head coach is Scott Frost, former offensive coordinator for the Oregon Ducks.', ' The Knights have played their home games at Bright House Networks Stadium located on the main campus of UCF in Orlando, Florida since 2007.']], ['UCF Knights football statistical leaders', ['The UCF Knights football statistical leaders are individual statistical leaders of the UCF Knights football program in various categories, including passing, rushing, receiving, total offense, defensive stats, and kicking.', ' Within those areas, the lists identify single-game, single-season, and career leaders.', \" The Knights represent the University of Central Florida in the NCAA's American Athletic Conference.\"]], ['2010 UCF Knights football team', ['The 2010 UCF Knights football team represented the University of Central Florida in the 2010 NCAA Division I FBS football season.', \" Their head coach was George O'Leary, who was in his seventh season with the team.\", ' For the first time in program history, the Knights were nationally ranked following a nationally televised rout of Houston on November 5.', ' For the third time in six years, UCF won the Conference USA Eastern Division and later, became Conference USA champions for the second time in four seasons.', ' As a result, the Knights appeared in the Liberty Bowl, in which they defeated Georgia 10–6, for the first bowl victory in program history.', ' UCF finished the season ranked in both final national polls, 20th in the Coaches Poll, and 21st in the AP Poll.']], ['2015 UCF Knights football team', ['The 2015 UCF Knights football team represented the University of Central Florida in the 2015 NCAA Division I FBS football season.', \" The Knights were members of the East Division of the American Athletic Conference (The American), and played their home games at Bright House Networks Stadium on UCF's main campus in Orlando, Florida.\", \" The Knights were led by head coach George O'Leary, who was in his 12th and final season with the team.\", \" After starting the season 0–6, O'Leary resigned as UCF's interim athletic director, a position he had held since June when Todd Stansbury left for the same position at Oregon State.\", \" Following UCF's 59–10 defeat by Houston on homecoming, dropping the Knights to an 0–8 record, O'Leary resigned as head football coach.\", ' Quarterbacks coach Danny Barrett served as interim head coach for the remainder of the season.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.584\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a70f0e75542994082a3e408', 'answer': 'with other campuses located in Chicago and Doha, Qatar', 'question': 'Which of these universities, Northwestern University or Johns Hopkins University, have a campus outside of the United States territories?', 'supporting_facts': [['Johns Hopkins University', 0], ['Northwestern University', 0]], 'context': [['Northwestern University', ['Northwestern University (NU) is a private research university based in Evanston, Illinois, with other campuses located in Chicago and Doha, Qatar, and academic programs and facilities in Washington, D.C., and San Francisco, California.']], ['Sara Berry', ['Sara Berry (born 1940) is a scholar of contemporary African political economies, professor of history at Johns Hopkins University and co-founder of the Center for Africana Studies at Johns Hopkins.', ' Berry received her PhD in economics at the University of Michigan in 1967 and has taught at Indiana University, Virginia Commonwealth University, Boston University, Johns Hopkins University, and Northwestern University.', ' Berry has published four books: \"Cocoa, Custom, and Socio-Economic Change in Rural Western Nigeria\" (1975, Oxford: Claredon) \"Accumulation, Mobility and Class Formation in an Extended Yoruba Community\" (1985, University of California Press), \"Boundries: Essays on Poverty, Power and the Past in Asante\", 1896-1996 (2001, Heinemann), and \"No Condition is Permanent: The Social Dynamics of Agrarian Change in Sub-Saharan Africa\" (1993, University of Wisconsin Press).', ' \"No Condition is Permanent\" won the 1985 Herskovits Prize for the year’s best book on Africa.', ' Berry has worked as a consultant for the Rockefeller Foundation, the Ford Foundation, the U.S. Agency for International Development.', ' The National Endowment for the Humanities, and the Herskovits Book Awards Committee.', ' She has received fellowships and awards from the Fulbright Senior Scholars Program, the Social Science Research Council, the Guggenheim Foundation, and the Mary Ingraham Bunting Institute at Radcliffe College.', ' Berry has a B.A. in history from Radcliffe College in 1961 and an M.A. from University of Michigan in 1965.']], ['Eric Sundquist', ['Eric Sundquist is an American scholar of the literature and culture of the United States.', ' Sundquist earned his B.A. from the University of Kansas (1974) and his Ph.D. from Johns Hopkins University (1978).', ' Sundquist is the Andrew W. Mellon Professor of the Humanities and former chair of the English Department at Johns Hopkins.', ' He is a former member of the UCLA Department of English, and was Dean of the College of Arts and Sciences at Northwestern University.']], ['Johns Hopkins School of Medicine', ['The Johns Hopkins University School of Medicine (JHUSOM), located in Baltimore, Maryland, U.S., is the academic medical teaching and research arm of Johns Hopkins University.', \" Johns Hopkins has consistently been among the nation's top medical schools in the number of research grants awarded by the National Institutes of Health.\", ' Its main teaching hospital, the Johns Hopkins Hospital, is ranked the #3 hospital in the United States by \"U.S. News & World Report\".']], ['Barton Childs', ['Barton Childs (February 29, 1916 – February 18, 2010) was an American pediatrician and geneticist.', ' He was born in Chicago, Illinois, and graduated from Williams College in 1938.', ' In 1942, he received his M.D. from Johns Hopkins University.', ' Following military service in World War II, he returned to Johns Hopkins for a residency in pediatrics.', ' After a fellowship at Children’s Hospital in Boston, he returned to Johns Hopkins University in 1949, where he remained until his retirement in 1981.', ' He remained a professor emeritus in the Department of Pediatrics at The Johns Hopkins University School of Medicine until his death.']], ['Richard A. Macksey', ['Richard A. Macksey (born 1931) is Professor of Humanities and Co-founder and longtime Director of the Humanities Center at The Johns Hopkins University, where he has taught critical theory, comparative literature, and film studies.', ' Professor Macksey was educated at Johns Hopkins, earning his B.A. in 1953 and his Ph.D. in 1957.', ' He has taught at Johns Hopkins (both the school of Arts & Sciences as well as the Medical School) since 1958.', ' He is the longtime Comparative Literature editor of MLN (Modern Language Notes), published by Johns Hopkins University Press.', ' He is a recipient of the Hopkins Distinguished Alumnus Award.', ' Dr. Macksey also presides over one of the largest private libraries in Maryland, with over 70,000 books and manuscripts.']], ['Johns Hopkins University', ['The Johns Hopkins University (commonly referred to as Johns Hopkins, JHU, or simply Hopkins) is an American private research university in Baltimore, Maryland.', ' Founded in 1876, the university was named for its first benefactor, the American entrepreneur, abolitionist, and philanthropist Johns Hopkins.', ' His $7 million bequest—of which half financed the establishment of Johns Hopkins Hospital—was the largest philanthropic gift in the history of the United States at that time.', \" Daniel Coit Gilman, who was inaugurated as the institution's first president on February 22, 1876, led the university to revolutionize higher education in the U.S. by integrating teaching and research.\", \" Adopting the concept of a graduate school from Germany's ancient Heidelberg University, Johns Hopkins University is considered the first research university in the United States.\"]], ['Bloomberg Distinguished Professorships', ['Bloomberg Distinguished Professorships (BDPs) were established as part of a $350 million gift by Michael Bloomberg, JHU Class of 1964, to Johns Hopkins University in 2013.', ' Fifty faculty members, ten from Johns Hopkins University and forty recruited from institutions worldwide, will be chosen for these endowed professorships based on their research, teaching, service, and leadership records.', ' The program is directed and managed by Johns Hopkins University Vice Provost for Research, Dr. Denis Wirtz.']], ['Daniel Webster (academic)', ['Daniel W. Webster (born 1960) is an American health policy researcher and the director of the Center for Gun Policy and Research at Johns Hopkins University.', ' He is also the deputy director for research at the Johns Hopkins Center for the Prevention of Youth Violence, and professor of Health Policy and Management at the Johns Hopkins Bloomberg School of Public Health.', ' In 2016, he became the director of the Johns Hopkins-Baltimore Collaborative for Violence Reduction, a joint crime-fighting effort between Johns Hopkins and the Baltimore Police Department.']], ['Johns Hopkins Berman Institute of Bioethics', ['The Johns Hopkins Berman Institute of Bioethics in Baltimore, Maryland, United States, is an independent, interdisciplinary center serving the entire Johns Hopkins University and Health System.', ' It is dedicated to the study of complex moral and policy issues in biomedical science, health care, and health policy.', ' Established in 1995, the Institute seeks answers to ethical questions by promoting research in bioethics and encouraging moral reflection among a broad range of scholars, professionals, students, and citizens.', ' Contributing to its mission are four divisions of the University: the Zanvyl Krieger School of Arts and Sciences, the Johns Hopkins School of Medicine, the Bloomberg School of Public Health, and the Johns Hopkins School of Nursing.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.587\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8e8af55542990e94052b3f', 'answer': 'Brown University', 'question': 'Which university is a private research university, University of California, Berkeley or Brown University?', 'supporting_facts': [['University of California, Berkeley', 0], ['Brown University', 0]], 'context': [['Brown University', ['Brown University is a private Ivy League research university in Providence, Rhode Island, United States.', ' Founded in 1764 as the College in the English Colony of Rhode Island and Providence Plantations, Brown is the seventh-oldest institution of higher education in the United States and one of the nine colonial colleges chartered before the American Revolution.']], ['Alpert Medical School', ['The Warren Alpert Medical School (formerly known as Brown Medical School, previously known as Brown University School of Medicine) is the medical school of Brown University, located in Providence, Rhode Island, United States.', ' Established in 1811, the school was among the first in the nation to offer academic medical education.', ' Today, Alpert Medical School is a component of Brown’s Division of Biology and Medicine, which also includes the Program in Biology.', ' (A third component of the Division, the Program in Public Health, became the Brown University School of Public Health on July 1, 2013.)', ' Together with the Medical School’s seven affiliated teaching hospitals, the Division attracts over $300 million in external research funding per year.']], ['Kaja Silverman', ['Kaja Silverman (born September 16, 1947) is an American art historian and critical theorist.', ' She is currently the Katherine and Keith L. Sachs Professor of Art History at the University of Pennsylvania.', ' She received her Ph.D. in English from Brown University.', ' Thereafter, she taught at Yale University, Trinity College, Simon Fraser University, Brown University, the University of Rochester and for many years was the Class of 1940 Professor in the Rhetoric Department at the University of California, Berkeley.', ' She was awarded a Guggenheim Fellowship in 2008, and is currently the holder of an Andrew W. Mellon Foundation Distinguished Achievement Award.']], ['Margaret Kidwell', ['Margaret Gale Kidwell (born August 17, 1933) is a British American evolutionary biologist and Regents’ Professor Emerita at the University of Arizona, Tucson.', ' She grew up on a farm in the English Midlands during World War II.', ' After graduating from the University of Nottingham in 1953, she worked in the British Civil Service as an Agricultural Advisory Officer from 1955-1960.', ' She moved to the USA in 1960 under the auspices of a Kellogg Foundation Fellowship to study Genetics and Statistics at Iowa State University.', ' She married quantitative geneticist James F. Kidwell in 1961, obtained her MS degree in 1962 and moved with her husband to Brown University in 1963.', ' She received her PhD from Brown University in 1973 under the guidance of Masatoshi Nei.', ' From 1973 to 1984 she pursued independent research into a number of anomalous genetic phenomena in \"Drosophila\" which later lead to collaborative studies resulting in the discovery of hybrid dysgenesis and the isolation of transposable P elements.', ' After appointment as Professor of Biology at Brown University in 1984 she moved to the University of Arizona in 1985 as Professor of Ecology and Evolutionary Biology.', ' Additional positions included Chair of the Interdisciplinary Genetics Program from 1988-1991 and Head of the Department of Ecology and Evolutionary Biology from 1992-1997.', ' Research at the University of Arizona has increasingly focused on the evolutionary significance of transposable genetic elements.', ' In 1996, she was the first woman from Arizona to be elected to the United States National Academy of Sciences.']], ['University of Southern California', ['The University of Southern California (USC or SC) is a private research university located in Los Angeles, California.', ' Founded in 1880, it is the oldest private research university in California.', \" USC has historically educated a large number of the region's business leaders and professionals.\", ' In recent decades, the university has also leveraged its location in Los Angeles to establish relationships with research and cultural institutions throughout Asia and the Pacific Rim.', ' An engine for economic activity, USC contributes $8 billion annually to the economy of the Los Angeles metropolitan area and California.']], ['Hamdard University', ['Hamdard University (Urdu: ) is a private research university with campuses in Karachi and Islamabad, Pakistan.', ' It was founded in 1991 by the renowned philanthropist Hakim Said of the Hamdard Foundation.', ' Hamdard is one of the first and the oldest private institutions of higher education in Pakistan.', ' In Karachi, Hamdard University is the largest private research university with a campus area of over 350 acres.']], ['University of California Pavement Research Center', ['UCPRC is the University of California Pavement Research Center.', ' It is located on the University of California Davis campus in Davis, California.', \" UCPRC has additional facilities at the University of California, Berkeley's Field Station in Richmond, Ca.\", ' They perform pavement engineering research in pavement structures and materials.', ' They are funded through public and private research grants.']], ['Charles P. Nott', ['Charles Palmer Nott (October 25, 1872 – December 30, 1954) was an American botanist and college football player and coach.', ' He was the head football coach at the University of California, Berkeley for one season, in 1897, compiling a record of 0–3–2.', ' He was also worked on the faculty at UC Berkeley.', ' Nott played college football at Brown University, where he was the captain of the Brown Bears football team in 1895.', ' Nott came to California in 1896 and served as a line coach under Frank Butterworth for the 1896 California Golden Bears football team.']], ['University of California, Berkeley', ['The University of California, Berkeley (also referred to as UC Berkeley, Berkeley, and Cal ) is a public research university located in Berkeley, California.', \" Founded in 1868, Berkeley is the oldest of the ten research universities affiliated with the University of California system (although UCSF was founded in 1864 and predates the establishment of the UC system) and is ranked as one of the world's leading research universities and the top public university in the United States.\"]], ['Brown University School of Engineering', ['The Brown University School of Engineering is the engineering school at Brown University, a private Ivy League research university located in Providence, Rhode Island.', ' The school offers both graduate and undergraduate study in the field.', ' Undergraduate students may declare their major in engineering as late as the end of their sophomore year.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.587\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae3c0245542990afbd1e1c3', 'answer': 'model', 'question': 'What other jobs did the actress Olivia Munn from Mortdecai have?', 'supporting_facts': [['Mortdecai (film)', 2], ['Olivia Munn', 0]], 'context': [['Freeloaders (film)', ['Freeloaders is an American ensemble comedy film directed by Dan Rosen and written by Rosen and singer Dave Gibbs.', ' The film is produced by the Broken Lizard comedy troupe and is independently financed.', ' \"Freeloaders\" stars Clifton Collins Jr., Josh Lawson, Kevin Sussman, Zoe Boyle, Nat Faxon, Warren Hutcherson, Jane Seymour, Olivia Munn, Dave Foley and Counting Crows lead singer Adam Duritz.', ' It follows a group of friends who find their luxurious lifestyle threatened when the rock star they freeload off decides to sell his home.']], ['Office Christmas Party', ['Office Christmas Party is a 2016 American Christmas comedy film directed by Josh Gordon and Will Speck and written by Justin Malen and Laura Solon, based on a story by Jon Lucas and Scott Moore.', ' The film stars an ensemble cast, including Jason Bateman, Olivia Munn, T. J. Miller, Jillian Bell, Vanessa Bayer, Courtney B. Vance, Rob Corddry, Kate McKinnon and Jennifer Aniston, and was released on December 9, 2016 by Paramount Pictures.', ' It grossed $114 million worldwide.']], ['Deliver Us from Evil (2014 film)', ['Deliver Us from Evil is a 2014 American supernatural horror film directed by Scott Derrickson and produced by Jerry Bruckheimer.', ' The film is officially based on a 2001 non-fiction book entitled \"Beware the Night\" by Ralph Sarchie and Lisa Collier Cool, and its marketing campaign highlighted that it was \"inspired by actual accounts\".', ' The film stars Eric Bana, Édgar Ramírez, Sean Harris, Olivia Munn, and Joel McHale in the main roles and was released on July 2, 2014.']], ['The Lego Ninjago Movie', ['The Lego Ninjago Movie is a 2017 3D computer-animated action comedy martial arts film produced by Warner Animation Group.', ' Co-directed by Charlie Bean, Paul Fisher and Bob Logan, the film was co-written by Logan, Fisher, William Wheeler, Tom Wheeler, Jared Stern and John Whittington.', ' The film stars the voices of Dave Franco, Justin Theroux, Fred Armisen, Abbi Jacobson, Olivia Munn, Kumail Nanjiani, Michael Peña, Zach Woods and Jackie Chan.', ' The story within a story focuses on Lloyd Garmadon, a teenage ninja, as he attempts to accept the truth about his villainous father, while a new threat emerges to endanger his homeland.']], ['Olivia Munn', ['Lisa Olivia Munn (born July 3, 1980) is an American actress and model.', ' She was credited as Lisa Munn in her early career, but since 2006, she has used the name Olivia Munn.']], ['Akira Fuse', ['Akira Fuse (布施 明 , Fuse Akira , born on December 18, 1947) is a Japanese singer, who was once married to Olivia Hussey.', ' He debuted in 1965 with the single \"Kimi ni Namida to Hohoemi o\" (君に涙とほほえみを , \"Tears and Smiles to You\") .', ' His greatest hits are \"Cyclamen no Kahori\" (シクラメンのかほり , Shikuramen no Kahori , \"The Scent of Cyclamen\") and \"Kimi wa Bara Yori Utsukushii\" (君は薔薇より美しい , \"You are More Beautiful than a Rose\") .', ' He currently makes consistent appearances on television, performs occasional seasonal tours, hosts a late night talk show, and is involved in a number of stage plays.', ' In 2005, he enjoyed a revival when his music was used in the popular Japanese TV-series \"Kamen Rider Hibiki\" with its ending theme \"Shōnen yo\" (少年よ , \"Boy!\")', ' and later its second opening theme \"Hajimari no Kimi e\" (始まりの君へ , \"To the Original You\") .', ' He married the actress Olivia Hussey in 1980 and later divorced in 1989 after he was unable to attain work in the United States and Olivia was unable to relocate her first-born son to Japan.', ' They had one child, son Maximillian Fuse, who currently attends university in America.']], ['The Babymakers', ['The Babymakers is a 2012 American comedy film directed by Jay Chandrasekhar, and starring Paul Schneider, Olivia Munn and Kevin Heffernan.', ' Chandrasekhar and Heffernan are both members of Broken Lizard.', ' The film received a limited release on August 3, 2012 in theaters and on video on demand services.', ' It received a DVD and Blu-ray release September 18, 2012.']], ['Desi Lydic', ['Lani Desmonet \"Desi\" Lydic (born June 30, 1981) is an American comedian and actress who is currently a correspondent on \"The Daily Show\" with Trevor Noah.', ' She also stars as guidance counselor Valerie Marks on the MTV comedy-drama series \"Awkward\".', ' She got her start in the 2001 parody film \"Not Another Teen Movie\".', ' She also starred in the Spike mini-series \"Invasion Iowa\" alongside William Shatner, and the parody series \"The Real Wedding Crashers\".', ' She appeared as Shea Seger in the 2011 film \"We Bought a Zoo\" alongside Matt Damon and Scarlett Johansson, and in the 2013 film \"The Babymakers\" with Olivia Munn.', ' Lydic also appeared as one half of a lesbian couple on an episode of the Disney Channel series \"Good Luck Charlie\" in 2014.', ' She joined Trevor Noah\\'s lineup of correspondents for \"The Daily Show\" on September 29, 2015.']], ['Ride Along 2', ['Ride Along 2 is a 2016 American action comedy film directed by Tim Story and written by Phil Hay and Matt Manfredi.', ' It is the sequel to the 2014 film \"Ride Along\".', ' The film stars Kevin Hart, Ice Cube, Ken Jeong, Benjamin Bratt, Olivia Munn, Bruce McGill and Tika Sumpter.', ' Universal Pictures released the film on January 15, 2016.', ' Like the original film, this sequel was panned by most critics but was a box office success, grossing $124.6 million worldwide during its theatrical run.']], ['Mortdecai (film)', ['Mortdecai is a 2015 American action comedy film directed by David Koepp and written by Eric Aronson.', ' The film is adapted from the novel series \"Mortdecai\" (specifically its first installment \"Don\\'t Point that Thing at Me\") written by Kyril Bonfiglioli.', ' It stars Johnny Depp in the title role and also features Gwyneth Paltrow, Ewan McGregor, Olivia Munn, Paul Bettany and Jeff Goldblum.', ' Released by Lionsgate on January 23, 2015, \"Mortdecai\" was a box office bomb, grossing $47 million against its $60 million budget, and received overwhelmingly negative reviews.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.588\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abe19895542991f661060c2', 'answer': 'China', 'question': 'Liaocheng and Shayang County are both located in which country?', 'supporting_facts': [['Liaocheng', 0], ['Shayang County', 0]], 'context': [['Country Club Mall', ['Country Club Mall is a shopping mall located in La Vale, Maryland, a suburb of Cumberland, Maryland in Allegany County, Maryland.', ' The mall has 60 retail units, as well as 7 vendor stands on the main concourse.', ' The largest retailers in the mall are Wal-Mart, the Bon-Ton, Sears, and JCPenney.', ' Also located in the Country Club Mall is the Country Club Mall 8 Cinemas, the largest movie theatre in Allegany County.', ' The mall is managed by Gumberg Asset Management Corp.']], ['Shayang County', [\"Shayang () is a county of west-central Hubei province, People's Republic of China.\", ' Administratively, it is part of the prefecture-level city of Jingmen.', ' The county is located south of the Jingmen city proper, west of the Han River, and north of the Chang Lake (\"Chang Hu\").']], ['DeSoto County, Mississippi', ['DeSoto County is a county located in the U.S. state of Mississippi.', ' As of the 2010 census, the population was 161,252, making it the third-most populous county in Mississippi.', ' Its county seat is Hernando.', ' DeSoto County is part of the Memphis, TN-MS-AR Metropolitan Statistical Area (MSA).', ' It is the second-most populous county in the MSA.', ' The county has lowland areas that were developed in the 19th century for cotton plantations, and hill country in the eastern part of the county.']], ['Liaocheng', ['Liaocheng (), also known as the Water City, is a prefecture-level city in western Shandong province, China.', \" It borders the provincial capital of Jinan to the southeast, Dezhou to the northeast, Tai'an to the south, and the provinces of Hebei and Henan to the west.\", ' The Grand Canal flows through the city center.', ' Its population was 5,789,863 at the 2010 census whom 1,229,768 lived in the built-up area made up of Donchangfu district, even though large parts remain rural.']], ['WSYY-FM', ['WSYY-FM (94.9 FM) is a radio station broadcasting for approximately 18¼ hours per day, 7 days a week (from 4:55AM through 11:10PM ET) under the slogan, \"\"Radio With An Attitude\"\".', ' Playing a mix of oldies/classic hits, adult contemporary, rock music, and some country crossovers, the station broadcasts an Adult Hits/Full-Service format for approximately 16 hours per day, from 6:00AM through 10:00PM ET (reserving the first and, also, the final hour of their broadcast day to \"When Radio Was\").', ' \"The Mountain 94.9\" carries local high school sports in season.', ' \"The Mountain 94.9\" had also carried the complete schedule of Red Sox Baseball (from 1997 through 2015, prior to becoming a Former Affiliate in 2016, which was when Millinocket\\'s affiliation with the Red Sox Baseball would ultimately be transferred over to co-owned WSYY-AM, thus concluding the frequent interruptions to the music on \"The Mountain 94.9\" during Baseball season).', ' The station currently features programming from CBS Radio and carries CBS Radio News at the top of every hour (and has been an affiliate of that network for many decades).', \" Licensed to Millinocket, Maine, United States, the station's broadcast signal serves the Central Penobscot County, Eastern Piscataquis County, and Southern Aroostook County Maine areas, and the station is licensed to serve the town of Millinocket, Maine, the very town where its studios/offices and tower site are located.\", ' The station is currently owned by Katahdin Communications, Inc.', ' WSYY-FM originally went on the air in 1978 on 97.7 FM as WKTR, upgrading to its current facilities in 1984 on 94.9.', ' Prior to their \"The Mountain 94.9\" branding, WSYY-FM used to be referred to as \"North Country 95\", airing a full-time Country Music format.', ' The current format, branding, and slogan was probably adopted around March 1, 2004, when Katahdin Communications, Inc. assumed control of WSYY-FM & WSYY-AM from Katahdin Timberlands, LLC (as a result of the radio station facing increasing land disputes), initially as a short term lease agreement but the transfer of ownership ultimately became permanent.', \" Those same land disputes would eventually lead to a loss of WSYY-FM's 23,500 watt transmitter location (featuring an antenna HAAT of 211 meters); as a result, WSYY-FM may have been operating under a Special Temporary Authority License (a 12,000 watt facility with an antenna HAAT of 68 meters via Hammond Ridge on Lake Road, about two miles from Millinocket Municipal Airport), ever since as long ago as late 2007, pending a planned permanent move to a 22,000 watt facility with an antenna HAAT of 198.4 meters (from just off Nicatou Road in Medway, well east of WSYY-FM's old or current transmitter tower location).\", ' On November 23th, 2016, the CP for this proposed move was modified to a 45,000 watt facility with an antenna HAAT of 146.7 meters, the first time this proposed move has ever received official approval from the FCC.', \" WSYY-FM is one of the two Maine affiliates—apart from WLOB—of When Radio Was (7 days a week from 5:00AM through 6:00AM ET and also from 10:00PM through 11:00PM ET), is one of the two Maine affiliates (WWMJ) of The Acoustic Storm (Saturdays from 9:00AM through 12:00PM ET), is Maine's only affiliate of the Crook & Chase syndicated Country Music countdown programming (Sunday afternoons from 2:00PM through 6:00PM ET), and is an affiliate of the Blues Deluxe radio show.\", \" WSYY-FM/WSYY-AM are unusual in that while these stations are authorized to broadcast 24 hours a day, the stations both have sign-offs every day (WSYY-AM signing off at sun-down, broadcasting only on Weekends (but not between Monday-through-Friday) and WSYY-FM broadcasting for approximately 18¼ hours per day, 7 days a week, WSYY-FM's broadcast day concluding with the 11:00PM ET Top-of-the-Hour CBS Radio newscast and then a Nightly Sign-Off Announcement and then an instrumentation of the American national anthem, followed by Dead Air amidst a Transmitter Power-Down, not Signing Back Onto The Air until 4:55AM ET).\", ' In Old Town and also Bangor (and continuing southward and/or southwestward), the station has strong FM co-channel interference with Portland-market WHOM (which transmits from atop Mount Washington in New Hampshire, the tallest peak in the Northeast and had for a long time claimed on its website that it has the largest coverage area of any FM station in the United States, its signal spanning five states: NH, ME, VT, MA, NY and also parts of Southern Quebec Province, Canada), this matter being especially problematic before dawn or after dusk.', ' In favorable atmospheric conditions, a very weak signal of WHOM can be DX-ed in Millinocket during overnight hours (when WSYY-FM is off-the-air).']], ['Quan (state)', ['The State of Quán () was a small Zhou Dynasty (1046–256 BC) vassal state of Central China.', ' A Marquisate, then Dukedom (侯), its rulers were descendants of Shang Dynasty (c. 1600–1046 BC) ruler Wu Ding with the surname \"Zi\" (子).', ' Quan was founded by Wen Ding’s son Quan Wending (权文丁) in the area of modern day Maliang Town (马良镇), Shayang County, Jingmen City, Hubei Province, next to what would later emerge as the State of Chu.']], ['Marion County, Missouri', ['Marion County is a county located in the northeastern portion of the U.S. state of Missouri.', ' As of the 2010 census, the population was 28,781.', ' Its county seat is Palmyra.', ' Unique from most third-class counties in the state, Marion has two county courthouses, the second located in Hannibal.', ' The county was organized December 23, 1826 and named for General Francis Marion, the \"Swamp Fox,\" who was from South Carolina and served in the American Revolutionary War.', ' The area was known as the \"Two Rivers Country\" before organization.']], [\"Dong'e County\", [\"Dong'e County falls under the jurisdiction of Liaocheng Prefecture-level city, in the Shandong Province of China.\", ' It is located on the left (northern) bank of the Yellow River, some 100\\xa0km upstream from the provincial capital Jinan.']], ['Telford Shopping Centre', ['Telford Shopping Centre is a 25 acre indoor super-regional shopping centre in Telford, Shropshire, England.', ' It is located in the geographical and economic centre of the new town, on land which was previously undeveloped.', ' It is the largest shopping area in the ceremonial county of Shropshire, being located roughly equidistant between Shrewsbury, the county town, and the West Midlands conurbation.', ' With a floor area of 100,000 m², the centre is one of the largest in the country, and has an average footfall of 300,000 per week, equating to 15 million per annum.', ' The centre is located on a 50 acre site, containing over 175 stores.', \" The Centre's catchment population is over 3 million people.\", ' The term Telford Town Centre is often used to refer to the shopping centre alone, but the town centre also encompasses the town park and surrounding areas of central Telford.', \" The centre's logo features The Iron Bridge, of nearby Ironbridge, a UNESCO World Heritage Site.\", ' In 2008 the centre was ranked as 14th best in the country by CACI.']], ['Clay County Historical Museum', ['The Clay County Historical Society Museum is located in Green Cove Springs, Clay County, Florida.', ' It is located in a former trail depot.', ' Exhibits include railroad memorabilia, a country kitchen display and a country store display.', ' It is operated by the Clay County Historical Society.', ' The museum is located at 915 Walnut Street in the Historical Triangle which also includes the 1896 county jail and 1890 courthouse at Walnut Street and Ferris Street (Hwy 16).']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.588\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a85e2775542994775f60680', 'answer': 'Kim Bauer', 'question': 'Just Before I Go stars an actress who plays what character in the series \"24\"?', 'supporting_facts': [['Just Before I Go', 0], ['Elisha Cuthbert', 1]], 'context': [['Elisha Cuthbert', ['Elisha Ann Cuthbert (born November 30, 1982) is a Canadian actress and model.', ' She became known for playing Kim Bauer in the series \"24\", Darcie Goldberg in the college comedy \"Old School\", Danielle in the teen comedy film \"The Girl Next Door\", and Carly Jones in the 2005 remake of \"House of Wax\".', ' In 2013, \"Maxim\" magazine named her \"TV\\'s most beautiful woman\".']], ['Just Before I Go', ['Just Before I Go, previously entitled Hello I Must Be Going, is a 2014 black comedy drama film directed by Courteney Cox, in her directorial debut, from a screenplay written by David Flebotte, starring Seann William Scott, Elisha Cuthbert, Olivia Thirlby, Garret Dillahunt, and Kate Walsh.']], ['Tori Go! Go!', ['Tori Go!', ' Go!', ' () is a South Korean animated television series.', ' It is a product of the major broadcaster KBS, and the animation was done by Duru Fix, Gangwon Information and Multimedia corporation, and DPS Corporation.', ' The story centers on the girl character Tori Go!', ' Go!', ', a high teen girls character squirrel.']], ['Holden Snyder and Lily Walsh', ['Holden Snyder and Lily Walsh Snyder are fictional characters and the signature supercouple of the CBS daytime soap opera \"As the World Turns\".', ' The role of Lily was first portrayed by actress Lucy Deakins from 1984 until her departure in 1985, when actress Martha Byrne joined the cast in 1985, which began the pairing of Lily with Holden opposite actor Jon Hensley.', ' Byrne departed the role of Lily in 1989, and it was recast with actress Heather Rattray.', ' In 1993, Rattray was let go from the series and Byrne returned to the role.', ' In 2008, Byrne made her high-profiled exit from the series and the role was once again recast with Noelle Beck, who remained in the role until the series finale in September 2010.', \" Hensley is the only actor to portray the role of Holden for the character's duration with the series.\"]], ['List of Yes! PreCure 5 episodes', ['\"Yes!', ' PreCure 5\" is the fourth \"Pretty Cure\" anime television series produced by Toei Animation.', ' The story is about a group of five girls who have the ability to transform into Pretty Cure.', ' They have been given this ability in order to collect the fifty-five Pinkies spread across the land and save the Palmier Kingdom.', ' The series began airing in Japan from February 4, 2007 and January 27, 2008, replacing \"Futari wa Pretty Cure Splash★Star\" in its initial timeslot and was replaced by its direct sequel series \"Yes!', ' PreCure 5 GoGo!\"', '.', ' The series uses three pieces of theme music, one opening and two ending themes.', ' The opening theme is \"PreCure 5, Smile Go Go!\"', ' (プリキュア5、スマイル go go!', ' Purikyua Faibu, Sumairu gō gō?)', ' performed by Mayu Kudou (Voice of Fairy Tone from Suite PreCure) with the chorus performed by Young Fresh with Mayumi+Yuka.', ' From episode 1-32, the ending theme is \"Kirakira-shichatte My True Love!\"', ' (キラキラしちゃってMy True Love!', ' Kirakira-shicatte Mai Turū Rabu?', ', \"Sparkle Brilliantly My True Love!\")', ' performed by Kanako Miyamoto (The Voice of Makoto Kenzaki/Cure Sword from Dokidoki!', ' PreCure).', ' From episode 33-49, the ending theme was changed to \"Ganbalance de Dance ~Yumemiri Kiseki-tachi~\" (ガンバランス de ダンス～夢みる奇跡たち～ Ganbaransu de Dansu ~Yumemiru Kiseki-tachi~\"?)', ', performed by Miyamoto with the PreCure 5.', ' This song was also used as the theme for the film adaptation Great Miraculous Adventure in the Mirror Kingdom!', '.', ' An insert song in the series titled \"Tobikkiri!', ' Yūki no Door\" (とびっきり!勇気の扉（ドア） Tobikkiri!', ' Yūki no Doa?', ', \"Extraordinary!', ' The Door of Courage\") was performed by Mariya Ise as her character Urara Kasugano and was used in episodes 20 and 29.']], ['United Red Army (film)', ['United Red Army (実録・連合赤軍 あさま山荘への道程 , Jitsuroku Rengōsekigun Asama-Sansō e no Dōtei ) is a 2007 film written, directed and produced by Kōji Wakamatsu.', \" It stars Akie Namiki as Hiroko Nagata and Go Jibiki as Tsuneo Mori, the leaders of Japan's leftist paramilitary group, the United Red Army.\", ' Akie Namiki was nominated for Best Performance by an Actress at the 2008 Asia Pacific Screen Awards.']], ['Farrah Phelan', ['Farrah Phelan (now Clayton) is a fictional character from the Irish television soap opera, \"Fair City\".', ' The character has been portrayed by four actresses in total.', ' The character first appeared onscreen in 1998 played by Fiona Glascott, however the character left the series to go to London.', ' On her return in 2000, Sinead Keenan was cast in the role.', ' The character left the series for a second time and when she returned for a third time, the role went to Denise McCormack.', ' The character again left the series to take up a photography course in London.', \" In 2014, Caroline Harvey was cast in the role when the character's father Christy Phelan had a stroke.\"]], ['Go! Princess PreCure', ['Go!', ' Princess PreCure (Go！プリンセスプリキュア , Gō!', ' Purinsesu PuriKyua ) , also known as Go!', ' Princess Pretty Cure, is a 2015 Japanese magical girl anime series produced by Toei Animation, and the twelfth installment to Izumi Todo\\'s \"Pretty Cure\" franchise.', ' It is directed by Yuta Tanaka and written by Hitoshi Tanaka of \"\" with character designs by Yukiko Nakatani.', ' The series began airing on February 1, 2015, succeeding \"HappinessCharge PreCure!', '\" in its initial timeslot.', \" The series's main theme is hopes and dreams with the cures' overall motif being princesses, keys and perfumes.\", ' It was then succeeded by Maho Girls PreCure!', ' on February 7, 2016.']], ['Go Man Go (film)', ['Go, Man, Go!', ' is a 1954 sports film directed by James Wong Howe, starring Dane Clark, Sidney Poitier, Ruby Dee, Patricia Breslin, The Harlem Globetrotters and Slim Gaillard.', ' Clark plays Abe Saperstein, the organizer of the Globetrotters.', \" Poitier's character is Inman Jackson, the team's showboating center.\", \" Breslin plays Sylvia Saperstein, the love interest, and Abe's daughter.\", ' Gaillard plays himself.']], ['Go Lala Go!', ['Go Lala Go!', ' () is a 2010 Chinese romantic comedy film about a Chinese woman who learns how to balance a relationship and professional work in a work place.', ' It is directed by Xu Jinglei, who also plays the title character, and is based on a novel, \"Du Lala\\'s Promotion\", by Li Ke.', ' The film also stars Stanley Huang and Karen Mok.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.589\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8b7d925542995d1e6f13bd', 'answer': 'comedy', 'question': '\"Horrible Bosses\" is written by a writer in which career?', 'supporting_facts': [['Michael Markowitz', 0], ['Horrible Bosses', 0]], 'context': [['Horrible Bosses 2', ['Horrible Bosses 2 is a 2014 American comedy film directed by Sean Anders and written by Anders and John Morris.', ' A sequel to 2011\\'s \"Horrible Bosses\", the film stars Jason Bateman, Charlie Day, Jason Sudeikis, Jennifer Aniston, Jamie Foxx, Chris Pine, and Christoph Waltz.', ' It was released on November 26, 2014 by Warner Bros.', ' Pictures.', ' The film grossed $107.7 million worldwide.']], ['Keeley Hazell', ['Keeley Rebecca Hazell (born 18 September 1986) is an English glamour model, actress.', ' Hazell became one of Britain\\'s most successful glamour models , working with brands such as Page 3, \"FHM\", \"Loaded\", \"Nuts\" and \"Zoo Weekly\".', ' She has also made numerous television appearances and has, more recently, focused on her acting career, appearing in films such as \"Horrible Bosses 2\".']], ['Horrible Bosses', ['Horrible Bosses is a 2011 American black comedy film directed by Seth Gordon, written by Michael Markowitz, John Francis Daley and Jonathan Goldstein, based on a story by Markowitz.', ' It stars Jason Bateman, Charlie Day, Jason Sudeikis, Jennifer Aniston, Colin Farrell, Kevin Spacey and Jamie Foxx.', ' The plot follows three friends, played by Bateman, Day and Sudeikis, who decide to murder their respective overbearing, abusive bosses, portrayed by Spacey, Aniston and Farrell.']], ['Jason Sudeikis', ['Daniel Jason Sudeikis ( ; born September 18, 1975) is an American actor, comedian and screenwriter.', ' He began his career in improv comedy.', ' In 2003, he was hired as a sketch writer for \"Saturday Night Live\" and was a cast member from 2005 to 2013.', ' He has appeared on television in \"30 Rock\", \"The Cleveland Show\", \"Eastbound & Down\", \"The Last Man on Earth\", and other shows.', ' He starred in the films \"Horrible Bosses\" (2011), \"Hall Pass\" (2011), \"We\\'re the Millers\" (2013), \"Horrible Bosses 2\" (2014), \"Sleeping with Other People\" (2015), \"Tumbledown\" (2015), \"The Book of Love\" (2016) and \"Race\" (2016).']], ['Brandon Richardson', ['Brandon Quantavius Richardson (born September 23, 1984) is an American actor.', ' He has played roles in \"Regular Show\", \"The Heat\", \"Meet the Browns\", \"\", \"Magic Mike XXL\", \"Horrible Bosses 2\" and \"Jurassic World\".', ' He is known for 2011 VH1 reality television \"Tough Love\" starring Steven Ward.']], ['Jonathan Goldstein (filmmaker)', ['Jonathan Michael Goldstein (born September 2, 1968) is an American screenwriter, television writer/producer, and film director.', ' He has written for numerous situation comedies, including \"The PJ\\'s\" starring Eddie Murphy, \"The Geena Davis Show\", \"Good Morning Miami\", \"Four Kings\", and \"The New Adventures of Old Christine\".', ' He is known for co-writing \"Horrible Bosses\" and \"\", and for co-writing and directing \"Vacation\" with his creative partner John Francis Daley.']], ['Southern Gothic (album)', ['Southern Gothic is the debut album by Atlanta-based hip hop/rock band The Constellations.', ' It was released on June 21, 2010 and features cameo appearances from Cee-Lo Green and Asher Roth.', ' \"Perfect Day\" has been featured in films and TV shows such as \"Horrible Bosses\", \"Chuck\" and \"Suits\".']], ['Jason Bateman', ['Jason Kent Bateman (born January 14, 1969) is an American actor, director, and producer.', ' He began acting on television in the early 1980s on \"Little House on the Prairie\", and in the sitcoms \"Silver Spoons\" and \"The Hogan Family\".', ' In the 2000s, he became known for his role of Michael Bluth using deadpan comedy in the critically acclaimed sitcom \"Arrested Development\", for which he won a Golden Globe and a Satellite Award.', ' He has had starring roles in the films \"Juno\" (2007), \"Hancock\" (2008), \"Up in the Air\" (2009), \"The Switch\" (2010), \"Paul\" (2011), \"Horrible Bosses\" (2011), \"The Change-Up\" (2011), \"Identity Thief\" (2013), \"Bad Words\" (2013), \"Horrible Bosses 2\" (2014), \"The Gift\" (2015), and \"Zootopia\" (2016), as well as the 2017 Netflix series \"Ozark\".']], ['Michael Markowitz', ['Michael Markowitz (born August 15, 1961) is a writer, producer, and actor who began his comedy career in The Mee-Ow Show, an improv group at Northwestern University.', ' Some projects he has worked on include \"Duckman\", \"Becker\", and the films \"Horrible Bosses\", \"Horrible Bosses 2\" and \"Boob Job\".', ' He has collaborated several times in the past with Jason Alexander.', ' As an actor, he appeared in the films \"The Flamingo Kid\" and \"Last Resort\", and the TV shows \"Becker\" and \"World Cup Comedy\".']], ['Brendan Hunt (actor)', ['Brendan Hunt is an American actor and writer known for roles in the films \"We\\'re the Millers\" (2013) and \"Horrible Bosses 2\" (2014) as well as voicing two characters in the video game \"Fallout 4\" (2015).']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.589\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abbc7a055429931dba1451b', 'answer': 'first', 'question': 'In what order does Three Colours: Blue appear in the Three Colours trilogy?', 'supporting_facts': [['Three Colours: Blue', 1], ['Three Colours trilogy', 0]], 'context': [['Three Colours: White', ['Three Colours: White (French: Trois couleurs : Blanc ) is a 1994 French-Polish comedy-drama film co-written, produced, and directed by Krzysztof Kieślowski.', ' \"White\" is the second in \"The Three Colors Trilogy\", themed on the French Revolutionary ideals, following \"\" and preceding \"\".', ' The film was selected as the Polish entry for the Best Foreign Language Film at the 67th Academy Awards, but was not accepted as a nominee.']], [\"Be All That You Can't Be\", ['\"Be All That You Can\\'t Be\" is the first single from Broadway Calls\\' second studio album, \"Good Views, Bad News\".', ' It was released on July 21, 2009.', ' The single has been released on vinyl.', ' The vinyl is available in three colours: Blue, orange and white (Hot Topic Exclusive).', ' Each colour is limited to 500.', ' The music video for the song was released through Absolute Punk on 6 August 2009.']], ['Colombia (cocktail)', ['The Colombia is a cocktail containing vodka and curaçao.', ' The layering effect takes advantage of the variation in density and temperature between the layers.', ' The drink appears as stacked horizontal layers of yellow, blue and red, which matches the three colours of the Colombian flag.']], ['Hokusai Manga', ['The Hokusai Manga (北斎漫画 , \"Hokusai\\'s Sketches\") is a collection of sketches of various subjects by the Japanese artist Hokusai.', ' Subjects of the sketches include landscapes, flora and fauna, everyday life and the supernatural.', ' The word \"manga\" in the title does not refer to the contemporary story-telling \"manga\", as the sketches in the work are not connected to each other.', ' Block-printed in three colours (black, gray and pale flesh), the Manga comprise literally thousands of images in 15 volumes, the first published in 1814, when the artist was 55.', ' The final three volumes were published posthumously, two of them assembled by their publisher from previously unpublished material.', ' The final volume was made up of previously published works, some not even by Hokusai, and is not considered authentic by art historians.']], ['Three Colours: Blue', ['Three Colours: Blue (French: Trois couleurs : Bleu ) is a 1993 French drama film directed and co-written by Polish filmmaker Krzysztof Kieślowski.', ' \"Blue\" is the first of three films that comprise the \"Three Colours\" trilogy, themed on the French Revolutionary ideals of liberty, equality, and fraternity; it is followed by \"\" and \"\".', ' According to Kieślowski, the subject of the film is liberty, specifically emotional liberty, rather than its social or political meaning.']], ['Blue', ['Blue is the colour between violet and green on the spectrum of visible light.', ' Human eyes perceive blue when observing light with a wavelength between 450 and 495 nanometres. Blues with a higher frequency and thus a shorter wavelength appear more violet, while those with a lower frequency and a longer wavelength gradually appear more green.', ' Pure blue, in the middle, has a wavelength of 470 nanometres.', ' In painting and traditional colour theory, blue is one of the three primary colours of pigments, along with red and yellow, which can be mixed to form a wide gamut of colours.', ' Red and blue mixed together form violet, blue and yellow together form green.', ' Blue is also a primary colour in the RGB colour model, used to create all the colours on the screen of a television or computer monitor.']], ['Three Colours trilogy', ['The \"Three Colours\" trilogy (Polish: \"Trzy kolory\" , French: \"Trois couleurs\" ) is a three-part film series directed by Krzysztof Kieślowski.', ' Two of the films were made in French and one primarily in Polish: \"\" (1993), \"\" (1994), and \"\" (1994).', ' All three were co-written by Kieślowski and Krzysztof Piesiewicz (with story consultants Agnieszka Holland and Sławomir Idziak) and have musical scores by Zbigniew Preisner.']], ['City of Sydney flag', ['The City of Sydney flag is made up of a horizontal triband of three colours – white, gold and blue.', ' It was designed in 1908.', ' The top third of the flag features three designs.', ' The flag is displayed in Town Hall, Sydney.']], ['Pan-African colours', ['The term Pan-African colours refers to two different sets of three colours: red, gold (not yellow), and green (inspired by the flag of Ethiopia), and red, black, and green.', ' They are used in flags and other emblems of various countries and territories in Africa and the Americas to represent Pan-Africanist ideology.', ' The Rastafarian movement and many Pan-African organisations also often employ the colours for their activities.']], ['Requiem for my friend (Preisner)', ['Requiem for my friend is a major and the first non-film musical work composed by Zbigniew Preisner.', ' The composition was meant to honour the composer\\'s late friend, the director Krzysztof Kieślowski, with whom he collaborated while working on a number of films, including the famous \"Three Colours\" trilogy.', ' The album was released in 1998 although some parts of the work must have been ready upon Kieślowski\\'s passing as Preisner asserted in an interview that \"the Requiem had accompanied Krzysztof in his last journey\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.590\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8651185542994775f60752', 'answer': 'yes', 'question': 'Are Fabián Bielinsky and Nelson Pereira dos Santos both directors?', 'supporting_facts': [['Fabián Bielinsky', 0], ['Nelson Pereira dos Santos', 0]], 'context': [['Tenda dos Milagres (film)', ['Tenda dos Milagres is a 1977 Brazilian drama film directed by Nelson Pereira dos Santos, based on the novel of the same name by Jorge Amado.', ' It stars Hugo Carvana, Sonia Dias and Severino Dada.', ' The director satirizes and exposes racism in Brazilian society.', ' The most notable example where this is done is in a flashback scene where Brazilians are shown listening with approving interest to Nazi race theories in the late 1930s.', ' \"Tenda dos Milagres\" was shot in Salvador, Bahia.']], ['José Manuel Bento dos Santos', ['José Manuel Bento dos Santos (born 1947) is a Portuguese cook, gastronomer, businessman, trader, metals broker, writer and chemical engineer.', ' Bento dos Santos studied chemical-industrial engineering at the Instituto Superior Técnico (IST), Technical University of Lisbon.', ' During his early university years he played rugby union and was also the manager of the Portugal national rugby team in 1967/1968.', ' He started his professional career as an employee of Companhia União Fabril (CUF), the largest Portuguese business conglomerate before the Carnation Revolution in 1974.', ' During the turmoil of the revolution, indeed a left-wing military coup, CUF was forcibly nationalized by the revolutionary government and collapsed.', ' Bento dos Santos, together with other partners such as Eduardo Catroga, founded the metals brokerage and trading company Quimibro.', ' In the late 1980s, Pedro Passos Coelho, future Prime Minister of Portugal, was invited by a cousin to work there as a collaborator.', ' Bento dos Santos studied also economics at the \"Instituto Superior de Economia e Gestão\" of the Technical University of Lisbon, but he did not graduate.', \" Beyond Quimibro, Bento dos Santos founded other ventures like Quinta do Monte d'Oiro, a winery.\", ' In the 2000s, Bento dos Santos became popular due to his cuisine progames \"O Sentido do Gosto\" (2007) on the Portuguese television RTP and \"Segredos do Vinho\" (SIC, 2004).', ' He also published a book of cuisine with the title \"O Sentido do Gosto\" as well as \"Subtilezas Gastronómicas – receitas à volta de um vinho\" (Assírio & Alvim, 2005).', ' He is affiliated with the International Gastronomy Academy, the Confrérie de la Chaîne des Rôtisseurs, the Confraria do Vinho do Porto, the Académie des Psycologues du Goût, and is a \"chevalier\" of both \"des Entonneurs Rabelaisiens\" and \"du Tastevin\".']], ['The Amulet of Ogum', ['The Amulet of Ogum (Portuguese: O Amuleto de Ogum ) is a 1974 Brazilian drama film directed by Nelson Pereira dos Santos.', ' It was entered into the 1975 Cannes Film Festival.', ' It was also selected as the Brazilian entry for the Best Foreign Language Film at the 48th Academy Awards, but was not accepted as a nominee.']], ['Tent of Miracles (novel)', ['Tent of Miracles (Portuguese: \"Tenda dos Milagres\" ) is a Brazilian Modernist novel.', ' It was written by Jorge Amado in 1967 and published the following year.', ' It was later adapted to a 1977 Cinema Novo (Nouvelle Vague) film by director/screenplay writer Nelson Pereira dos Santos.']], ['How Tasty Was My Little Frenchman', ['How Tasty Was My Little Frenchman (Portuguese: \"Como Era Gostoso o Meu Francês\" ) is a Brazilian black comedy directed by Nelson Pereira dos Santos released in 1971.']], ['Fabián Bielinsky', ['Fabián Bielinsky (3 February 1959 – 28 June 2006) was an Argentine film director born in Buenos Aires.']], ['Jubiabá (film)', ['Jubiabá (French: Bahia de tous les saints ) is a 1986 Brazilian-French romantic drama film directed by Nelson Pereira dos Santos.', ' Based on the novel of the same name by Jorge Amado, it stars Charles Baiano and Françoise Goussard as two lovers.']], ['Nelson Pereira dos Santos', ['Nelson Pereira dos Santos (born 22 October 1928) is a Brazilian film director.', ' He directed films such as \"Vidas Secas\" (Barren Lives), based on the book with the same name by Brazilian writer Graciliano Ramos.']], ['Rio, 100 Degrees F.', ['Rio, 100 Degrees F \"(Portuguese: Rio, 40 Graus\") is\\xa0a 1955 Brazilian film written and directed by\\xa0Nelson Pereira dos Santos.', \" It is dos Santos' first feature work, inspired by the Italian Neo-Realism, and is considered a precursor of the Cinema Novo\\xa0movement.\"]], ['Institut des hautes études cinématographiques', ['L\\'Institut des hautes études cinématographiques (IDHEC; the \"Institute for Advanced Cinematographic Studies\") is a French film school, founded during World War II under the leadership of Marcel L\\'Herbier who was its president from 1944 to 1969.', ' IDHEC offered training for directors and producers, cameramen, sound technicians, editors, art directors and costume designers.', ' It became highly influential, and many prominent film-makers received their training there including Paulo Rocha, Louis Malle, Alain Resnais, Claire Denis, Volker Schlöndorff, Jean-Jacques Annaud, Claude Sautet, Nelson Pereira dos Santos, Patrice Leconte, Costa Gavras, Theo Angelopoulos, Omar Amiralay, Rithy Panh, Arnaud Desplechin, Claude Miller, Alfonso Gumucio Dagron Christopher Miles and Pascale Ferran.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.591\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab850b255429916710eb036', 'answer': 'bat and ball', 'question': 'What items are used to play both Lapta and Rounders?', 'supporting_facts': [['Lapta (game)', 0], ['Rounders', 0]], 'context': [['Japanese tea ceremony utensils', ['Equipment for tea ceremony is called \"dōgu\" (道具; lit., \"tools\"), or more specifically \"chadōgu\" (茶道具; \"tea tools\").', ' \"Chadōgu\" can be divided into five major categories: \"sōshoku dōgu\" (装飾道具; \"decorative items\"); \"temae dōgu\" (点前道具; \"items for the tea-making and service\"); \"kaiseki dōgu\" (懐石道具; \"items for the \"chakaiseki\" meal\"); \"mizuya dōgu\" (水屋道具; \"items used in the preparation room\"); and \"machiai dōgu / roji\" \"dōgu\" (待合道具・露地道具; \"items for the waiting room\" / \"items for the \"roji\" garden\").', ' A wide range of \"dōgu\" is necessary for even the most basic tea ceremony.', ' Generally, items which guests prepare themselves with for attending a chanoyu gathering are not considered as \"chadōgu\"; rather, the term fundamentally applies to items involved to \"host\" a chanoyu gathering.', ' This article, however, includes all forms of implements and paraphernalia involved in the practice of \"chanoyu\".']], ['Indexed unit of account', ['When a daily indexed unit of account or Daily Consumer Price Index (Daily CPI) or monetized daily indexed unit of account is used in contracts or in the Capital Maintenance in Units of Constant Purchasing Power accounting model, deferred payments and constant real value non-monetary items are indexed to the general price level in terms of a Daily Index such that changes in the inflation rate - in the case of monetary items - and the stable measuring unit assumption - in the case of constant real value non-monetary items - have no effect on the real value of these items.', ' Non-indexed units, such as contracts written in nominal currency units and nominal monetary items, incur inflation or deflation risk in the case of monetary items.', ' During all periods of inflation (low, high or hyperinflation), the debtor pays less in real terms than what both the debtor and creditor agreed at the original time of the contract/sale.', ' On the other hand, in periods of deflation, the debtor pays more in real terms than the original agreed value.', ' The opposite is true for creditors.', ' Contracts and constant real value non-monetary items accounted in daily indexed units of account, Daily CPI or monetized daily indexed units of account incur no inflation or deflation risk, as the real value of payments and outstanding capital amounts remain constant over time while the nominal values are inflation- or deflation-indexed daily.']], ['Lapta (game)', ['Lapta (Russian: лапта́ ) is a Russian bat and ball game first known to be played in the 14th century.', ' Mentions of lapta have been found in medieval manuscripts, and balls and bats were found in the 14th-century layers during excavations in Novgorod.', ' It is similar to cricket, brännboll, Rounders, baseball, oină, (Italy) and pesäpallo.']], ['Bayley Scales of Infant Development', ['The Bayley Scales of Infant and Toddler Development (Bayley-III is the current version) is a standard series of measurements originally developed by psychologist Nancy Bayley used primarily to assess the development of infants and toddlers, ages 1–42 months.', ' This measure consists of a series of developmental play tasks and takes between 45\\xa0– 60 minutes to administer and derives a developmental quotient (DQ) rather than an intelligence quotient (IQ).', ' Raw scores of successfully completed items are converted to scale scores and to composite scores.', \" These scores are used to determine the child's performance compared with norms taken from typically developing children of their age (in months).\", ' The most recent edition, the Bayley-III has three main subtests; the Cognitive Scale, which includes items such as attention to familiar and unfamiliar objects, looking for a fallen object, and pretend play, the Language Scale, which taps understanding and expression of language, for example, recognition of objects and people, following directions, and naming objects and pictures, and the Motor Scale, which assesses gross and fine motor skills such as grasping, sitting, stacking blocks, and climbing stairs.', ' There are two additional Bayley-II Scales depend on parental report, including the Social-Emotional scale, which asks caregivers about such behaviors as ease of calming, social responsiveness, and imitation play, and the Adaptive Behavior scale which asks about adaptions to the demands of daily life, including communication, self-control, following rules, and getting along with others.', ' The Bayley-III Cognitive and Language scales are good predictors of preschool mental test performance.', ' These scores are largely used for screening, helping to identify the need for further observation and intervention, as infants who score very low are at risk for future developmental problems.']], ['Takeshi Gear', ['Takeshi Gear are items used in the Japanese \"tokusatsu\" television series, \"Kamen Rider Hibiki\".', ' The various Takeshi Gear are both ancient items as well as items using technology to advance the mystical power of the \"Oni Kamen\" Riders.', ' The Takeshi Gear is modeled after several Japanese and non-Japanese objects.', ' Ranging from tuning forks to whistles, these are some of the more distinctive transformation items used in any Kamen Rider series.']], ['Rounders', ['Rounders (Irish: \"cluiche corr\" ) is a bat-and-ball game played between two teams.', ' Rounders is a striking and fielding team game that involves hitting a small, hard, leather-cased ball with a rounded end wooden, plastic or metal bat.', ' The players score by running around the four bases on the field.']], ['Functional item', [\"In the framework of Noam Chomsky's Minimalist Program, items of the lexicon are of two types: with or without substantive content.\", ' Items of the former category are called lexical items, whereas items of the latter category are functional items.', ' Functional items carry the grammatical content of a sentence, which means that by taking them out of the sentence one would still understand the meaning, although it would not be grammatical.', \" In other words, they are the 'glue' that holds the sentence together.\", ' Functional items can also be classified as closed class, that is, belonging to parts of speech that do not easily allow new members.', ' If functional items are removed from a sentence, the words that would be left are the lexical items.', ' The lexical items of a sentence are those that are used in telegraphic speech; functional items are the grammatical units that hold the sentence together and make it more fluid.', ' Functional Items are feature sets.', ' Functional items include two type of morphemes.', ' Free morphemes, like modals, auxiliaries, determiners, complementizers, and bound morphemes such as nominal and verbal affixes.', ' Though functional items have feature structure, the do not enter into θ-marking.']], ['Brännboll', ['Brännboll (] ) (brennball or slåball (\"hitball\") in Norway; rundbold (\"roundball\") in Denmark; Brennball in Germany) is a game similar to rounders, baseball, lapta and pesäpallo played on amateur level throughout Sweden, Norway, Denmark and Germany, mostly on fields and in public parks, but it is also part of the PE curriculum in some areas.', ' The name is derived from the act of catching a player between two bases at the end of a batting round, referred to as \"burning\" them (\"bränna\"), roughly equivalent to being run out in cricket or out in baseball.', ' The world championship, called Brännbollscupen, is an annual event in the Swedish city of Umeå.']], ['Home accessories', [\"Home accessories are furniture items which are easy to replace and easy to move, and include almost any items that aren't strictly functionally necessary in the decorated space.\", ' These accessories include such items as curtains, sofa sets, cushions, tablecloths and decorative craft products, decorative wrought iron, and so on.', ' These items are commonly used in indoor furnishings and layout and can include cloth items, paintings, and plants.']], ['Pesäpallo', ['Pesäpallo (] ; Swedish: \"boboll\" , both names literally meaning \"nest ball\", also referred to as \"Finnish baseball\") is a fast-moving bat-and-ball sport that is often referred to as the national sport of Finland and has some presence in other countries including Germany, Sweden, Switzerland, Australia, and Canada\\'s northern Ontario (the latter two countries have significant Nordic populations).', ' The game is similar to brännboll, rounders, and lapta, as well as baseball.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.592\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac279345542990b17b153b0', 'answer': '28 January 1864', 'question': 'What is the date of birth of the English inventor that developed the Richard Hornsby & Sons oil engine? ', 'supporting_facts': [['Richard Hornsby &amp; Sons', 1], ['Herbert Akroyd Stuart', 0]], 'context': [['Richard Newsham', ['Richard Newsham (died 1743) was an English inventor.', ' He took out 2 patents for fire engines in 1721 and 1725 (Royal Patent Office 1721 patent #439 and 1725 patent #479) and soon dominated the fire engine market in England.', ' The engine had two single-acting pistons and an air vessel placed in a tank which formed the frame of the machine.', ' The pump was worked by people at the long cross handles.', ' At the front of the engine, protected by a sheet of horn and a door, were directions for keeping the machine in order.', ' The cistern could hold about 170 USgal of water pumping up to 100 USgal a minute.', ' New York City imported its first two fire engines from Newsham in 1731.', ' In 1737 Newsham made a manual fire pump for the Parish of Bray in Berkshire.']], ['Timothy Hornsby', ['Timothy Richard Hornsby CBE (born 22 September 1940) is British.', ' He is Chairman of the Horniman Museum 2004–present.', ' He is the son of Harker William Hornsby']], ['Herbert Akroyd Stuart', ['Herbert Akroyd-Stuart (28 January 1864, Halifax, Yorkshire, England – 19 February 1927, Halifax) was an English inventor who is noted for his invention of the hot bulb engine, or heavy oil engine.']], ['Richard Hornsby', ['Richard Hornsby Elsham in Lincolnshire 4 June 1790 - 1864.', 'was an inventor and founder of a major agricultural machinery firm that developed steam engines.', ' His firm also developed early diesels and caterpillar tracks.', ' He lived with a farming family, the son of William Hornsby and his wife Sarah.']], ['St Vincents Hall', ['St Vincents Hall, Grantham, is a Gothic Revival mansion built in 1868 for the industrialist Richard Hornsby who founded Richard Hornsby & Sons, engine and machinery manufacturer.']], ['Associated British Oil Engine Company', ['The Associated British Oil Engine Company (ABOE) was a British engineering company.', ' It started life as a combine, similar to Agricultural & General Engineers.', ' Petters Limited joined ABOE in 1937.', ' J&H McLaren & Co. was sold to ABOE in 1943, although it may have been a member from an earlier date.', ' In 1945 Mirrlees, Bickerton and Day joined the group followed by the National Gas and Oil Engine company in 1950.']], ['Mount Hornsby', ['Mount Hornsby ( ) is a prominent snow-capped mountain on the south side of the middle reaches of Sjogren Glacier, Trinity Peninsula, Antarctica.', ' It was mapped from surveys by the Falkland Islands Dependencies Survey (1960–61), and was named by the UK Antarctic Place-Names Committee after Richard Hornsby & Sons of Grantham, who designed and constructed several highly successful chain-track vehicles for the British War Office, the first \"caterpillar tractors,\" in the years 1904–10.']], ['Crude oil engine', ['The crude oil engine is a type of internal combustion engine similar to the hot bulb engine.', ' A crude oil engine could be driven by all sorts of oils such as engine waste oil and vegetable oils.', ' Even peanut oil and butter could be used as fuel if necessary.', ' Like hot bulb engines, crude oil engines were mostly used as stationary engines or in boats.', ' They can run for a very long time; for instance, at the world fair in Milan in 1906, a FRAM engine was started and ran until the exhibition was over one month later.', ' A crude oil engine is a low RPM engine dimensioned for constant running and can last for a very long time if maintained properly.', ' It was later replaced by the diesel engine.']], ['Oil engine', ['An oil engine is an internal combustion engine that is powered by the burning of fuel oil, as opposed to external combustion engines, such as steam engine.', ' The term usually refers to low compression engines, so the diesel engine is usually not included.']], ['Richard Hornsby &amp; Sons', ['Richard Hornsby & Sons was an engine and machinery manufacturer in Lincolnshire, England from 1828 until 1918.', ' The company was a pioneer in the manufacture of the oil engine developed by Herbert Akroyd Stuart, which was marketed under the \"Hornsby-Akroyd\" name.', ' The company developed an early track system for vehicles, selling the patent to Holt & Co. (predecessor to Caterpillar Inc.) in America.', ' In 1918, Richard Hornsby & Sons became a subsidiary of the neighbouring engineering firm Rustons of Lincoln, to create \"Ruston & Hornsby\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.592\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac471305542991943173998', 'answer': 'prevent the opposing team from scoring goals', 'question': 'What is the primary role of the position played by Viktor Raychev for Lokomotiv Sofia?', 'supporting_facts': [['Viktor Raychev', 0], ['Defender (association football)', 0]], 'context': [['Viktor Raychev', ['Viktor Raychev (Bulgarian: Виктор Райчев ; born 26 May 1986) is a Bulgarian footballer who plays as a defender for Lokomotiv Sofia.']], ['WBC Lokomotiv Sofia', [\"WBC Lokomotiv Sofia (Bulgarian: ВБК „Локомотив София“ ) are a Bulgarian women's basketball club based in the capital Sofia and part of the Lokomotiv Sofia sports club.\"]], ['1982 Bulgarian Cup Final', ['The 1982 Bulgarian Cup Final was the 42nd final of the Bulgarian Cup (in this period the tournament was named Cup of the Soviet Army), and was contested between Lokomotiv Sofia and Lokomotiv Plovdiv on 12 June 1982 at Slavi Aleksiev Stadium in Pleven.', ' Lokomotiv Sofia won the final 2–1 after extra time.']], [\"Bulgarian Women's Basketball Championship\", [\"The Bulgarian Women's Basketball Championship is the premier league for women's basketball clubs in Bulgaria.\", \" Slavia Sofia is the championship's most successful club with 15 titles between 1953 and 2004 followed by Akademik Sofia and Levski Sofia with eight titles, Lokomotiv Sofia with seven and Minyor Pernik with six, while Neftokhimik Burgas has been the most successful team in recent years with 5 titles since 2005.\"]], ['1948 Bulgarian Cup Final', ['The 1948 Bulgarian Cup Final was the 8th final of the Bulgarian Cup (in this period the tournament was named Cup of the Soviet Army), and was contested between Lokomotiv Sofia and Slavia-Chengelov on 9 May 1948 at Yunak Stadium in Sofia.', ' Lokomotiv won the final 1–0.']], ['1953 Bulgarian Cup Final', ['The 1953 Bulgarian Cup Final was the 13th final of the Bulgarian Cup (in this period the tournament was named Cup of the Soviet Army), and was contested between Lokomotiv Sofia and Levski Sofia on 25 November 1953 at Vasil Levski National Stadium in Sofia.', ' Lokomotiv won the final 2–1, claiming their second national cup title.']], ['BC Lokomotiv Sofia', ['BC Lokomotiv Sofia (Bulgarian: БК „Локомотив София“ ) are a Bulgarian basketball club based in the capital Sofia and part of the Lokomotiv Sofia sports club.']], ['1995 Bulgarian Cup Final', ['The 1995 Bulgarian Cup Final was played at the Vasil Levski National Stadium in Sofia on May 27, 1995, and was contested between the sides of Lokomotiv Sofia and Botev Plovdiv.', ' The match was won by Lokomotiv Sofia.']], ['Lokomotiv Sofia (sports club)', ['Lokomotiv Sofia is a sports club from Sofia, Bulgaria, founded in 1929.', ' Its football team, PFC Lokomotiv Sofia, is its most renowned sports branch.']], ['Defender (association football)', ['In the sport of association football, a defender is an outfield player whose primary role is to prevent the opposing team from scoring goals.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.593\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab33ef955429969a97a810e', 'answer': 'Washington', 'question': 'Corn Ranch is a spaceport where test flights are carried out by a company headquartered in what state?', 'supporting_facts': [['Corn Ranch', 0], ['Blue Origin', 0]], 'context': [['Corn Ranch', ['Corn Ranch is a spaceport in the West Texas town of Van Horn, Texas, where flight tests of the New Shepard are carried out by Blue Origin.', ' The 165,000-acre (670 km²) land parcel was purchased by Internet billionaire Jeff Bezos.', ' The first flight test took place on November 13, 2006 with the goal of providing commercial tourist flights.']], ['Project Highwater', [\"Project Highwater was an experiment carried out as part of two of the test flights of NASA's Saturn I launch vehicle (using battleship upper stages), successfully launched into a sub-orbital trajectory from Cape Canaveral, Florida.\", ' The Highwater experiment sought to determine the effect of a large volume of water suddenly released into the ionosphere.', ' The project answered questions about the effect of the diffusion of propellants in the event that a rocket was destroyed at high altitude.']], ['H-T Ranch', ['H-T Ranch is a historic ranch complex 10 mi west of Amidon, North Dakota.', ' The ranch originally consisted of ten buildings; however, only the ranch house and log barn survive.', ' The ranch served as the headquarters of the Little Missouri Cattle Company, which was run by Arthur Clark Hidekoper.', ' Hidekober established the ranch in the 1880s, and by the end of the decade, it had become the \"most notable\" ranch in the area.', ' The ranch house, called Shackford, was built in 1896; its uncommon style resembles a bungalow but also borrows from other styles such as the Stick style.', ' By 1906, the \"Fargo Forum\" described the ranch as \"the biggest and most important [ranch] in the state\" and \"one of the largest horse raising outlets in the world\".', ' In the same year, Hidekoper sold the 70,000 acre ranch; the sale was the largest land deal in North Dakota history.', ' After the sale, a land company reduced the ranch to 5000 acre ; it was later used as a dude ranch in the 1920s.']], ['De Havilland Sprite', ['The Sprite was a British rocket engine built by de Havilland for use in RATO (Rocket-assisted take off) applications.', ' For RATO use only a short burn time is required, with simplicity and light weight as major virtues.', ' The intended market was for assisting take-off of de Havilland Comet 1 airliners (as hot and high operations in the British Empire were considered important) and also for V bombers carrying heavy nuclear weapons.', ' 30 successful test flights were carried out by Comets, from May 1951, but gas turbine performance improved rapidly, and so RATO was not required in service.']], ['Blue Origin', ['Blue Origin is an American privately funded aerospace manufacturer and spaceflight services company set up by Amazon.com founder Jeff Bezos with its headquarters in Kent, Washington.', ' The company is developing technologies to enable private human access to space with the goal to dramatically lower costs and increase reliability.', ' Blue Origin is employing an incremental approach from suborbital to orbital flight, with each developmental step building on its prior work.', ' The company motto is \"\"Gradatim Ferociter\"\", Latin for \"Step by Step, Ferociously\".', ' Blue Origin is developing a variety of technologies, with a focus on rocket-powered Vertical Takeoff and Vertical Landing (VTVL) vehicles for access to suborbital and orbital space.', \" The company's name refers to the blue planet, Earth, as the point of origin.\"]], ['Apollo 7', ['Apollo 7 was an October 1968 human spaceflight mission carried out by the United States.', \" It was the first mission in the United States' Apollo program to carry a crew into space.\", ' It was also the first U.S. spaceflight to carry astronauts since the flight of Gemini XII in November 1966.', ' The AS-204 mission, also known as \"Apollo 1\", was intended to be the first manned flight of the Apollo program.', ' It was scheduled to launch in February 1967, but a fire in the cabin during a January 1967 test killed the crew.', ' Manned flights were then suspended for 21 months, while the cause of the accident was investigated and improvements made to the spacecraft and safety procedures, and unmanned test flights of the Saturn V rocket and Apollo Lunar Module were made.', \" Apollo 7 fulfilled Apollo 1's mission of testing the Apollo Command/Service Module (CSM) in low Earth orbit.\"]], ['SpaceShipOne flight 15P', ['Flight 15P of SpaceShipOne (X0) was the first privately funded human spaceflight.', ' It took place on June 21, 2004.', ' It was the fourth powered test flight of the Tier One program, the previous three test flights having reached much lower altitudes.', ' The flight carried only its pilot, Mike Melvill, who thus became the first non-governmental astronaut.']], ['Soyuz (rocket)', ['The Soyuz (Russian: Союз , meaning \"union\", GRAU index 11A511) was a Soviet expendable carrier rocket designed in the 1960s by OKB-1 and manufactured by State Aviation Plant No. 1 in Kuybyshev, Soviet Union.', ' It was commissioned to launch Soyuz spacecraft as part of the Soviet human spaceflight program, first with 8 unmanned test flights, followed by the first 19 manned launches.', ' The original Soyuz also propelled four test flights of the improved Soyuz 7K-T capsule between 1972 and 1974.', ' In total it flew 30 successful missions over 10 years and suffered two failures.']], ['John Christiansen', ['John “Chris” Christiansen (May 1, 1923 － September 12, 1998) was the chief military test pilot for Lockheed California Company for over 30 years.', \" He might be most known for having performed Lockheed S-3 Viking's maiden flight on January 21, 1972.\", ' His assignments also included the initial test flights of Lockheed P-3 Orion.', ' Christiansen was born in Oslo, Norway in 1923 and became an American citizen in 1939.', ' He later served in the U.S. Navy during World War II and the Korean Conflict.', ' He began experimental flying for Lockheed Martin in 1953, and worked there until his retirement in 1984.', ' He was a fellow at Society of Experimental Test Pilots.']], ['Orion abort modes', [\"NASA's newest spacecraft, the Orion Multi-Purpose Crew Vehicle (MPCV), will be the first American spacecraft since Project Apollo to use an escape system in the event of a launch abort, something its predecessor, the Space Shuttle, had for only its first four orbital test flights in 1981-1982.\", ' Like the Apollo Command-Service Module (CSM), the Orion CEV will use the Launch escape system (LES), a solid-fueled tractor rocket that will be able to pull the Orion crew module away from a malfunctioning Space Launch System (SLS) rocket during the initial launch phase.', ' Based on the launch escape system found on the Soviet/Russian Soyuz spacecraft, the LAS, designed and manufactured by ATK for the Orion CEV, will be larger than the Soyuz version and will have more thrust than the Atlas 109-D booster that carried astronaut John Glenn into orbit in 1962.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.594\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac19bf55542994ab5c67d97', 'answer': 'Joe McCoy and Memphis Minnie', 'question': 'High Water is a song by Bob Dylan it was based on the 1927 Lousiana Flood. Another song recorded by Dylan about the event is When the Levee Breaks who is the Husband and Wife team that wrote and recorded this song in 1929 ?', 'supporting_facts': [['High Water (For Charley Patton)', 1], ['High Water (For Charley Patton)', 3], ['When the Levee Breaks', 0]], 'context': [['To Ramona', ['\"To Ramona\" is a folk waltz written by Bob Dylan for his fourth studio album, \"Another Side of Bob Dylan\".', ' The melody is taken from traditional Mexican folk music.', ' \"To Ramona\" is also a nod to Rex Griffin\\'s 1937 song \"The Last Letter\".', \" The song is one of several on the album to highlight the more personal, less political, side of Dylan's songwriting that would become more prominent in the future.\", \" The song also makes allusions to Dylan's personal relationship with fellow folk singer Joan Baez, at the time of its composition and subsequent release.\", ' It is another example of the G, G6, G7 harmonic motif Dylan uses pervasively on the record.']], ['High Water (For Charley Patton)', ['\"High Water (For Charley Patton)\" is a song by Bob Dylan, released on his 31st studio album \"\"Love and Theft\"\" in 2001.', ' The song draws its title from the Charley Patton song \"High Water Everywhere\", and is meant as a tribute to that bluesman.', ' It is one of many songs based on the 1927 Louisiana flood.', ' Other songs about the event include Memphis Minnie\\'s \"When the Levee Breaks\" (also recorded by Dylan on his \"Modern Times\" album as \"The Levee\\'s Gonna Break\"), and Randy Newman\\'s \"Louisiana 1927\".']], [\"Just Like Tom Thumb's Blues\", ['\"Just Like Tom Thumb\\'s Blues\" is a song written and performed by Bob Dylan.', ' It was originally recorded on August 2, 1965, and released on the album \"Highway 61 Revisited\".', ' The song was later released on the compilation album \"Bob Dylan\\'s Greatest Hits Vol.', ' II\" and as two separate live versions recorded at concerts in 1966: the first of which appeared on the B-side of Dylan\\'s \"I Want You\" single, with the second being released on \"\".', ' The song has been covered by many artists, including Gordon Lightfoot, Nina Simone, Barry McGuire, Judy Collins, Frankie Miller, Linda Ronstadt, the Grateful Dead, Neil Young, The Black Crowes, and Bryan Ferry.', \" Lightfoot's version was recorded only weeks after Dylan's original had been released and reached #3 on the national RPM singles chart.\", ' In addition, the song was sampled by the Beastie Boys for their song \"Finger Lickin\\' Good.\"']], ['All I Really Want to Do', ['\"All I Really Want to Do\" is a song written by Bob Dylan and featured on his Tom Wilson-produced 1964 album, \"Another Side of Bob Dylan\" (\"see\" 1964 in music).', ' It is arguably one of the most popular songs that Dylan wrote in the period immediately after he abandoned topical songwriting.', ' Within a year of its release on \"Another Side of Bob Dylan\", it had also become one of Dylan\\'s most familiar songs to pop and rock audiences, due to hit cover versions by Cher and the Byrds.']], [\"Don't Hurt Yourself (Beyoncé song)\", ['\"Don\\'t Hurt Yourself\" is a song recorded by American singer Beyoncé for her sixth studio album, \"Lemonade\".', ' The song was produced by Jack White, Beyoncé and Derek Dixie, and written by White, Beyoncé, and Diana Gordon.', ' It contains samples of \"When the Levee Breaks\" written by Jimmy Page, Robert Plant, John Paul Jones and John Bonham, and performed by Led Zeppelin.', ' Some critics compared the song to Beyoncé\\'s \"Ring the Alarm\" (2006).', ' The song received a nomination for the 59th Annual Grammy Awards in the category Best Rock Performance. \"', 'Billboard\" ranked \"Don\\'t Hurt Yourself\" at number 61 on their \"\"Billboard\"\\' s 100 Best Pop Songs of 2016\" list.']], ['High water mark', ['A high water mark is a point that represents the maximum rise of a body of water over land.', ' Such a mark is often the result of a flood, but high water marks may reflect an all-time high, an annual high (highest level to which water rose that year) or the high point for some other division of time.', ' Knowledge of the high water mark for an area is useful in managing the development of that area, particularly in making preparations for flood surges.', ' High water marks from floods have been measured for planning purposes since at least as far back as the civilizations of ancient Egypt.', ' It is a common practice to create a physical marker indicating one or more of the highest water marks for an area, usually with a line at the level to which the water rose, and a notation of the date on which this high water mark was set.', ' This may be a free-standing flood level sign or other marker, or it may be affixed to a building or other structure that was standing at the time of the flood that set the mark.']], ['Shelter from the Storm', ['\"Shelter from the Storm\" is a song by Bob Dylan, released on his 15th studio album, \"Blood on the Tracks\", in 1975.', ' Along with \"Tangled Up in Blue\", \"Shelter from the Storm\" was one of two songs from \"Blood on the Tracks\" to be re-released on the 2000 compilation \"The Essential Bob Dylan\".', ' The song also appears on two live albums by Bob Dylan — \"Hard Rain\" (from a May 1976 performance) and \"At Budokan\" (recorded in February 1978).', ' A first take of the song, from the same recording session that produced the album track, is included on \"The Best of Bob Dylan, Vol.', ' 1\" (1997).']], ['When the Levee Breaks', ['\"When the Levee Breaks\" is a blues song written and first recorded by husband and wife Kansas Joe McCoy and Memphis Minnie in 1929.', ' The song is in reaction to the upheaval caused by the Great Mississippi Flood of 1927.']], ['Mama, You Been on My Mind', ['\"Mama, You Been on My Mind\" is a song by American singer-songwriter Bob Dylan.', ' Written in 1964 during a trip to Europe, the song dealt with his recent breakup with his girlfriend, Suze Rotolo.', ' Dylan first recorded the song in June of that year during a session for his album \"Another Side of Bob Dylan\".', \" However, the song was not included on the album, and Dylan's version remained unreleased until 1991.\", ' In total, in the 1990s and 2000s four versions were put out on Dylan\\'s \"Bootleg Series\" of releases, including two live performances with Joan Baez from 1964 and 1975.']], ['Early Spring 2008 Midwest floods', ['The March 2008 Midwest floods were a massive flooding event in the Southern Midwest and portions of the Southern Plains.', ' Cape Girardeau, Missouri officially reported 11.48 in between March 18 and 19.', ' At least 17 people died as a result of the flooding.', ' Levee breaks were observed in several areas, most notably in Southeastern Missouri, where levee breaks occurred through mid-April.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.595\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac4f7e955429924173fb523', 'answer': 'Sophy', 'question': 'The English chef known for his back-to-basics philosphy has a sister named what?', 'supporting_facts': [['Jane Fearnley-Whittingstall', 2], ['Hugh Fearnley-Whittingstall', 0]], 'context': [['Kenny Atkinson (chef)', [\"Kenny Atkinson, (born December 1976), is an English chef, who won Michelin stars at the restaurants St Martin's on the Isle, and Kenny Atkinson at the White Room within Seaham Hall.\", ' He has appeared on the BBC show \"Great British Menu\", where he has twice been selected as the chef for courses at the final banquets.', ' He was named Chef of the Year at the 2009 Catey Awards.', ' In 2015 he was award a Michelin star for his own restaurant House of Tides in Newcastle upon Tyne.']], ['Roberto Treviño', ['Roberto Treviño is an American chef known for his appearances on \"Iron Chef: America\" and \"The Next Iron Chef\".', ' He is chef and owner of three restaurants and a bar in the Condado area of San Juan, Puerto Rico.']], ['Jane Fearnley-Whittingstall', ['Jane M. Fearnley-Whittingstall (\"née\" Lascelles) (born 1939 in Kensington, London) is a writer and garden designer with a diploma in landscape architecture.', ' She has won two gold medals at Chelsea Flower Show.', ' She has two children: Sophy and Hugh, the celebrity chef, and six grandchildren.', ' From 2005 to 2007 she wrote a weekly column about family life, in \"The Times\".', ' She has also written for \"The Daily Telegraph\", \"Daily Mail\", \"The Oldie\", \"Woman\\'s Weekly\", \"The Garden\", \"The English Garden\" and \"Gardens Illustrated\".']], ['Garima Poddar', ['Garima Poddar (Hindi: गरिमा पोद्दार ; born 3 April 1997) is an Indian celebrity chef known for her title which she won IIHM Young Chef India schools.', ' Also she is known as Garry after (Garry Mehigan), her favourite judge in Masterchef Australia.', ' At the age of 17 she was honoured by Dr. Suborno Bose, as he draped a national flag across her shoulder.', ' Garima was the Torch Bearer in Young Chef Olympiad .']], ['Eudes Assis', ['Eudes Assis (from Boiçucanga, São Sebastião, São Paulo, Brazil) known as Chef Eudes, is a chef known in the Brazil for using ingredients of \"caiçara\" culture.']], ['Hugh Fearnley-Whittingstall', ['Hugh Christopher Edmund Fearnley-Whittingstall (born 14 January 1965) is an English celebrity chef, television personality, journalist, food writer and campaigner on food and environmental issues, known for his back-to-basics philosophy.']], ['Daniel Clifford (chef)', ['Daniel Clifford is an English chef who is best known for his work at the two Michelin star restaurant Midsummer House.', ' He was also named one of the winners of the 2012 and 2013 series of the BBC television show the \"Great British Menu\".', ' He is chef patron of a gastro pub in Little Dunmow, Essex, named The Flitch of Bacon.']], ['Geof Manthorne', ['Geoffrey Manthorne (born April 25, 1974), more commonly known as Geof, is an American chef known for his skill in cake building, as well as decorating.', ' He stars on the Food Network\\'s reality-TV show \"Ace of Cakes\" and works as executive sous chef at Duff Goldman\\'s bakery Charm City Cakes in Baltimore, Maryland.']], ['Bella Jakubiak', ['Isabella (Bella) Jakubiak is a self-taught Australian chef known for winning the 2011 series of reality television cooking programme, My Kitchen Rules with her sister Sammy.']], ['Boy Logro', ['Pablo Logro, popularly known as Boy Logro or Chef Boy (born June 29, 1956) is a Filipino celebrity chef known for his cooking shows, \"Idol sa Kusina\" and \"\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.596\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5aba3bb855429955dce3ede7', 'answer': '1934', 'question': \"Chien-Shiung Wu's expertise in experimental physics evoked comparisons to a scientist who passed away in which year ?\", 'supporting_facts': [['Chien-Shiung Wu', 4], ['Marie Curie', 0]], 'context': [['Marie Curie', ['Marie Skłodowska Curie ( ; ] ; ] ; 7 November 18674 July 1934; born Maria Salomea Skłodowska; ] ) was a Polish and naturalized-French physicist and chemist who conducted pioneering research on radioactivity.', ' She was the first woman to win a Nobel Prize, the first person and only woman to win twice, the only person to win a Nobel Prize in two different sciences, and was part of the Curie family legacy of five Nobel Prizes.', ' She was also the first woman to become a professor at the University of Paris, and in 1995 became the first woman to be entombed on her own merits in the Panthéon in Paris.']], ['Open information extraction', ['In natural language processing, open information extraction (OIE) is the task of generating a structured, machine-readable representation of the information in text, usually in the form of triples or n-ary propositions.', ' A proposition can be understood as truth-bearer, a textual expression of a potential fact (e.g., \"Dante wrote the Divine Comedy\"), represented in an amenable structure for computers [e.g., (\"Dante\", \"wrote\", \"Divine Comedy\")].', ' An OIE extraction normally consists of a relation and a set of arguments.', ' For instance, (\"Dante\", \"passed away in\" \"Ravenna\") is a proposition formed by the relation \"passed away in\" and the arguments \"Dante\" and \"Ravenna\".', ' The first argument is usually referred as the subject while the second is considered to be the object.']], ['2752 Wu Chien-Shiung', ['2752 Wu Chien-Shiung (1965 SP) is a main-belt asteroid discovered on 20 September 1965 by Purple Mountain Observatory at Nanking.', ' It was named for noted Chinese-American nuclear physicist Chien-Shiung Wu and was the first asteroid to be named after a living scientist.']], ['Allan Blaer', ['Allan Blaer (born 1942) is a physicist, Professor Emeritus and Special Lecturer at Columbia University in New York City.', ' He received his undergraduate degree from Columbia University in 1964, where he was the valedictorian.', ' He later went on to obtain his PhD in physics at the same institution.', ' He has done research in both theoretical and experimental physics.', ' In quantum field theory, he worked on phase transitions in low-temperature bosonic and fermionic systems, quantum field theory anomalies, dyons and magnetic monopoles in non-abelian gauge theories, and renormalization theory.', ' In experimental physics, he has worked on a precision measurement of vacuum polarization in muonic atoms to test quantum electrodynamics.']], ['Chien-Shiung Wu College', ['Chien-Shiung Wu College is a college named after lady Chien-Shiung Wu, an alumna of previous National Central University.', ' The college is a part of Southeast University(SEU), Nanjing.']], ['Abou Diaby', ['Vassiriki Abou Diaby (] ; born 11 May 1986), known as Abou Diaby, is a French professional footballer who is currently a free agent.', ' He plays primarily in a box to box role, as he is adept in both attacking and defending, and is described as a player who is \"languid, elusive, and athletic\" that can either \"dribble past opponents or slip passes to team-mates\".', ' Of Ivorian descent, Diaby also possesses \"superb touch\" and \"excellent close control\".', \" Diaby's career has been hampered by numerous repetitive injuries, a problem that has existed since his time in France.\", ' His physical appearance and positional preference have evoked comparisons to Arsenal legend and compatriot Patrick Vieira.']], ['Cui Youfu', ['Cui Youfu (崔祐甫) (721 – July 7, 780), courtesy name Yisun (貽孫), was an official of the Chinese dynasty Tang Dynasty, serving as a chancellor briefly early during the reign of Emperor Dezong.', ' He was credited for governing in an effective manner and guiding Emperor Dezong to correct decisions that, for some time, evoked comparisons between Emperor Dezong and his well-regarded ancestors Emperor Taizong and Emperor Xuanzong.']], ['Caleb Stine', ['Caleb Stine is an American singer/songwriter from Baltimore, Maryland.', ' He plays both solo and with Americana-rockers The Brakemen.', ' His timeless style and intensely personal lyrics has evoked comparisons to Townes Van Zandt, Kris Kristofferson, and Harvest-era Neil Young.']], ['Chien-Shiung Wu', ['Chien-Shiung Wu (; May 31, 1912 – February 16, 1997) was a Chinese-American experimental physicist who made significant contributions in the field of nuclear physics.', ' Wu worked on the Manhattan Project, where she helped develop the process for separating uranium metal into uranium-235 and uranium-238 isotopes by gaseous diffusion.', ' She is best known for conducting the Wu experiment, which contradicted the hypothetical law of conservation of parity.', ' This discovery resulted in her colleagues Tsung-Dao Lee and Chen-Ning Yang winning the 1957 Nobel Prize in physics, and also earned Wu the inaugural Wolf Prize in Physics a mere two decades later in 1978.', ' Her expertise in experimental physics evoked comparisons to Marie Curie.', ' Her nicknames include \"the First Lady of Physics\", \"the Chinese Madame Curie\", and the \"Queen of Nuclear Research\".']], ['Wu experiment', ['The Wu experiment was a nuclear physics experiment conducted in 1956 by the Chinese American physicist Chien-Shiung Wu in collaboration with the Low Temperature Group of the US National Bureau of Standards.', ' The experiment\\'s purpose was to establish whether or not conservation of parity (\"P\"-conservation), which was previously established in the electromagnetic and strong interactions, also applied to weak interactions.', ' If \"P\"-conservation were true, a mirrored version of the world (where left is right and right is left) would behave as the mirror image of the current world.', ' If \"P\"-conservation were violated, then it would be possible to distinguish between a mirrored version of the world and the mirror image of the current world.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.597\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae641f65542992ae0d162a3', 'answer': 'Charice', 'question': 'What is the mononym of the Fillipino singer that released the album \"My Inspiration\"?', 'supporting_facts': [['My Inspiration', 0], ['Jake Zyrus', 0]], 'context': [['Danilla Riyadi', ['Danilla Riyadi (known professionally by the mononym Danilla) is an Indonesian singer.', ' She has released just one studio album so far: Telisik (2014), and has appeared in several compilation albums, notably for the Tiga Dara remake soundtrack where she sung \"\"Bimbang Tanpa Pegangan\"\" and \"\"Pilih Menantu\"\" with Indra Aziz.', ' Her debut album was placed on number #18 of Rolling Stone Indonesia album of the year.']], ['Humood AlKhudher', ['Humood Othman AlKhudher (in Arabic حمود عثمان الخضر; born 1989), sometimes known only by his mononym Humood is a Kuwaiti singer.', ' He had a successful solo career, first as a child singer, and then as an independent artist.', ' He released his debut album \"Fekra\" in 2013.', ' In 2015 he was signed to Awakening Records and launched his album \"Aseer Ahsan\" with the label.', ' The album includes 10 uplifting songs produced by Awakening Records.']], ['My Inspiration', ['My Inspiration is a full-length studio album, released in the Philippines only, by Filipino singer Jake Zyrus.', ' It was released on May 1, 2009 and is credited under the name Charice, which Zyrus used prior to his gender transition.', ' It is a concept cover album dedicated to his mother Racquel Pempengco.', ' However, despite its ten cover versions (one of which is a remix), this second album by Zyrus features one original song (in both a single and extended version), called \"Always You\", written by the Metropop Song Festival and Awit Award-winning songwriter Jonathan Manalo.', ' The album has been certified platinum.', ' Like his previous release, the album was also made available for digital download through Amazon.com MP3 Download on June 23, 2009.', ' Some package versions include a second CD, which features a karaoke version of every song, very popular in Asian countries.']], ['Holy Smoke (Gin Wigmore album)', ['Holy Smoke is the debut album from New Zealand pop singer Gin Wigmore, released under the mononym Gin.', ' Singles released off the album included \"Oh My\", \"I Do\" and \"Hey Ho\".', ' The Cardinals play on every track and backed Wigmore on her subsequent tour.']], ['Jake Zyrus', ['Jake Zyrus (born Charmaine Clarice Relucio Pempengco; May 10, 1992), who performed under the mononym Charice until his gender transition to male, is a Filipino singer who rose to popularity through YouTube.']], ['Banks (singer)', ['Jillian Rose Banks (born June 16, 1988), known professionally by the mononym Banks (often stylized as BANKS), is an American singer and songwriter from Orange County, California.', ' She is signed to Harvest Records and Good Years Recordings.', ' Following the release of two extended plays, \"Fall Over\" and \"London\", in 2013, Banks released her debut album, \"Goddess\", on September 5, 2014, to positive reviews from contemporary music critics.', ' It reached number 12 on the US Billboard 100, while its most successful single, \"Beggin for Thread\", was certified gold by the RIAA.', ' Her second studio album, \"The Altar\", was released on September 30, 2016, to a similar positive reception.']], ['Janna Hurmerinta', ['Janna-Mari Hurmerinta (born 20 December 1981), better known by her mononym Janna, is a Finnish R&B, and pop singer, songwriter and pianist.', ' Her debut album \"Right Now\" was released in June 2007 to critical acclaim.', ' followed by \"The Makings of Me\" in 2008.', ' In 2013, she was signed to Universal Music Finland.', ' Her self-titled album \"Janna\" topped the Finnish album chart in June 2014.']], ['Myra (singer)', ['Mayra Caról Ambriz Quintana (born May 21, 1986 in Los Angeles, California)—better known under the mononym, Myra— is the daughter of Salvador Ambriz and Consuelo Quintana.', ' Myra is an American singer, dancer and choreographer of Mexican descent.', ' She is the first Latina artist to have been signed to Hollywood Records, Walt Disney Records, and Avex Trax.', ' She rose to prominence in 2001 after recording a cover of Martha and the Vandellas\\' \"Dancing in the Street\" for the soundtrack to the 2001 film \"\" (which she would then re-record in 2002 in Spanish for the Disney Channel original movie, Gotta Kick It Up!', ').', ' However, her first album released before being signed to Disney -released in 1997, was the Mariachi album \"Mensajera del Amór\", released by Briaz Promotions.', ' At the time of its release, Myra was then known as Mayra Caról.', ' During her Disney-era portion of her career, she contributed her vocals to the song \"Miracles Happen (When You Believe)\" to the film \"The Princess Diaries\", and it received a 2002 ALMA Award nomination for Outstanding Song in a Motion Picture Soundtrack.', ' Her debut album \"Myra\" included these two tracks and were released as singles and both included a music video.', ' The album \"Myra\" was released in four different formats.']], ['Karen (singer)', ['Karen Rosenberg (born 20 June 1975), better known by her mononym Karen is a Danish R&B singer.', ' She released three albums.', ' Her debut album \"En til en\" in 2000 was produced by Saqib of Outlandish and Lasse Lindholm of Hvid Sjokolade.', ' Her follow-up album.', ' The album was nominated to three awards during the Danish Music Awards eventually winning Best R&B.', ' She also became famous with \"Vis mig du\\' min mand\" taken from the album.', ' Her follow-up album \"Ingen smalle steder\" in 2004 was produced by her boyfriend producer Vagn Luv.', ' In 2009, she released \"Stiletto\", but with much lesser success.', \" the album didn't chart on the Tracklisten.\"]], ['Jeremih', ['Jeremy Felton (born July 17, 1987), better known by his mononym Jeremih ( ), is an American singer, songwriter, rapper and record producer.', ' In 2009, he signed a record deal with Def Jam Recordings.', ' Jeremih\\'s commercial debut single, \"Birthday Sex\", peaked at number four on the US \"Billboard\" Hot 100 chart.', ' His self-titled debut album reached number six on the US \"Billboard\" 200 chart.', ' Jeremih\\'s success continued with the release of his second album, \"All About You\", led by the single \"Down on Me\", which also reached the top five of the \"Billboard\" Hot 100.', ' In 2014, his single \"Don\\'t Tell \\'Em\" became his third top-ten hit on the \"Billboard\" Hot 100.', ' After multiple delays, Jeremih released his third studio album, \"Late Nights\" in 2015.', ' He announced that he is working on a joint album with PartyNextDoor called \"Late Night Party\".', ' He is also working on his fourth studio album, \"Later That Night\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.597\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae72fc5554299572ea54770', 'answer': 'Jaipur', 'question': 'Mundawar is a city 140km from which State capital of Rajasthan?', 'supporting_facts': [['Mundawar', 2], ['Jaipur', 0]], 'context': [['Jaipur', ['Jaipur ( ) is the capital and the largest city of the Indian state of Rajasthan in Western India.', ' It was founded on 18 November 1727 by Jai Singh II, the ruler of Amer after whom the city is named.', ' As of 2011, the city has a population of 3.1 million, making it the tenth most populous city in the country.', ' Jaipur is also known as the \"Pink City of India\".', ' Jaipur is located 280 km from the Indian capital New Delhi.', ' Jaipur forms a part of the west Golden Triangle tourist circuit along with Agra (240 km ).', ' Jaipur is a popular tourist destination in India and serves as a gateway to other tourist destinations in Rajasthan such as Jodhpur (348 km ), Jaisalmer (571 km ) Udaipur (421 km ).']], ['Siya Khoh', ['Siya Khoh is a small village in the Alwar District of Rajasthan, India.', ' It comes under the Mundawar tehsil, the Basmenpur gram panchayat, and the Jaipur Division.', ' It is located 45 km towards North from District headquarters Alwar, and 160 km from State capital Jaipur.', ' The PIN code of Siya Khoh is 301407, and thje postal head office is Menpur.']], ['Cholang', ['Cholang is a village in Jalandhar district of Punjab State, India.', ' It is located 39.7\\xa0km away from Phillaur, 16.4\\xa0km from district headquarter Jalandhar and 140\\xa0km from state capital Chandigarh.', ' The village is administrated by a sarpanch who is an elected representative of village as per Panchayati raj (India).']], ['Khanpur Ahir', ['Khanpur Ahir is a village in Mundawar Mandal in Alwar District in the Indian state of Rajasthan.', ' Khanpur Ahir is 40\\xa0km far from its District Headquarter Alwar.', ' It is 140\\xa0km far from its State Capital Jaipur.']], ['Mundawar', ['Mundawar (Hindi: मुंडावर) is a Tehsil in Alwar District in Rajasthan State.', ' Mundawar is 40 km far from its District Headquarter Alwar.', ' It is 140  km far from its State Capital Jaipur.', ' Mundawar part of Ahirwal region.']], ['Abapura', ['Abapura is a village panchayat located in the Banswara district of Rajasthan state,India.Abapura is a Village in Banswara Tehsil in Banswara District of Rajasthan State, India.', ' It belongs to Udaipur Division .', ' It is located 14\\xa0km towards South from District headquarters Banswara.', ' 16\\xa0km from Banswara.', ' 474\\xa0km from State capital Jaipur.The other nearest state capital from Abapura is Gandhinagar and its distance is 193.4\\xa0km.There is no railway station near to Abapura in less than 10\\xa0km.', ' How ever Ratlam Jn Rail Way Station is major railway station 60\\xa0km near to Abapura.', ' The nearest railway station to Abapura is Bhairongarh which is located in and around 41.9 kilometer distance.', ' The following table shows other railway stations and its distance from Mamakudi.']], ['Uppal Bhupa', ['Uppal Bhupa is a village in Jalandhar district of Punjab State, India.', ' It is located 8.8\\xa0km from postal head office in Nurmahal, 19.3\\xa0km from Phillaur, 37.3\\xa0km from district headquarter Jalandhar and 140\\xa0km from state capital Chandigarh.', ' The village is administrated by a sarpanch who is an elected representative of village as per Panchayati raj (India).']], ['Daduwal', ['Daduwal (Punjabi: ਦਾਦੂਵਾਲ ) is a village in Rurka Kalan tehsil in Jalandhar district of Punjab State, India.', ' It is located 12\\xa0km away from Phagwara, 36\\xa0km from Phillaur, 20.2\\xa0km from district headquarter Jalandhar and 140\\xa0km from state capital Chandigarh.', ' The village is administrated by a sarpanch who is an elected representative of village as per Panchayati raj (India).']], ['Mehsampur', ['Mehsampur (Punjabi: ਮਹਿਸਮਪੁਰ ) is a village in Jalandhar district of Punjab State, India.', ' It is located 16\\xa0km from Nakodar, 31\\xa0km from Phillaur, 40\\xa0km from district headquarter Jalandhar and 140\\xa0km from state capital Chandigarh.', ' The village is administrated by a sarpanch who is an elected representative of village as per Panchayati raj (India).']], ['Akalpur', ['Akalpur (Punjabi: ਅਕਲਪੁਰ ) is a village in Phillaur tehsil of Jalandhar District of Punjab State, India.', ' It is 2\\xa0km from Phillaur, 45.8\\xa0km from Jalandhar, and 112\\xa0km from state capital Chandigarh.', ' The nearest train station is situated in Phillaur, nearest domestic airport is 33\\xa0km away in Ludhiana and the nearest international airport is 140\\xa0km away in Amritsar.', ' The village is administrated by Sarpanch who is elected representative of village and it has postal head office 2\\xa0km away in Phillaur.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.598\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae0ead25542990adbacf6c8', 'answer': 'Jersey barrier', 'question': 'What is the name for plastic water-filled barriers that you might find along the MacArthur Road highway in Pennsylvania?', 'supporting_facts': [['Pennsylvania Route 145', 0], ['Pennsylvania Route 145', 5], ['Jersey barrier', 4]], 'context': [['Salty liquorice', ['Salty liquorice, also known as salmiak or salmiakki (in Finland), is a variety of liquorice flavoured with ammonium chloride, common in the Nordic countries, the Netherlands, and northern Germany.', ' Ammonium chloride gives salty liquorice an astringent, salty taste (hence the name), which has been described as \"tongue-numbing\" and \"almost-stinging\".', ' Salty liquorice is an acquired taste and people not familiar with ammonium chloride might find the taste physically overwhelming and unlikeable.', ' Salty liquorice candies are almost always black or very dark brown and can range from very soft to very hard and may be brittle.', ' The other colours used are white and variants of grey.', ' Salty liquorice is also used as a flavouring in other products, such as ice creams and alcoholic beverages.']], ['Fundamental articles (theology)', ['Fundamental articles was a term employed by early Protestant theologians, who wished to distinguish some essential parts of the Christian faith from non-essential doctrines.', ' There were then a number of reasons for establishing such a distinction.', ' Individual churches might accept or reject parts of doctrine, without forfeiting their claim to rank as parts of the universal Church.', ' Therefore, theologians might find a dogmatic basis for union between separated churches.', ' Also, the arguments of Catholics could be answered in a polemical way.', ' To Protestants, the fundamental articles are those that Christians must believe to be saved.']], ['Johor Bahru Inner Ring Road', ['Johor Bahru Inner Ring Road, (JBIRR) or Jalan Lingkaran Dalam, Federal Route 188, is a multi-lane federal ring road highway that circles the Johor Bahru Central District in Johor, Malaysia.', ' The 4.6 km highway was the most expensive road infrastructure project per kilometre of its time in the country.']], ['Jersey barrier', ['A Jersey barrier or Jersey wall is a modular concrete or plastic barrier employed to separate lanes of traffic.', ' It is designed to minimize vehicle damage in cases of incidental contact while still preventing the crossover case of a head-on collision.', ' Jersey barriers are also used to reroute traffic and protect pedestrians and workers during highway construction, as well as temporary and semi-permanent protections against landborne attack such as suicide vehicle bombs.', ' A Jersey barrier is also known in the western United States as K-rail, or K-wall, a term borrowed from the California Department of Transportation specification for temporary concrete traffic barriers, or colloquially as a Jersey bump.', ' Plastic water-filled barriers of the same general shape are also now commonly called Jersey barriers.']], ['Smozhe', ['The village is located along the highway road Highway M06 (Ukraine) (M06 ), on the southern slopes of the ridge (998 – 1056 m).']], ['Ontario Highway 622', ['Secondary Highway 622, commonly referred to as Highway 622, is a long connecting route between Highways 11 and Highway 17 in Northwestern Ontario.', ' The road is connected to Highway 11 by the 3\\xa0km-long spur road Highway 11B in Atikokan.']], ['Breadon Field', ['Breadon Field was a minor league ballpark in Whitehall Township, Pennsylvania, located on the east side of MacArthur Road, about 0.5 mile north of the U.S. Route 22 interchange.']], ['Pennsylvania Route 145', ['Pennsylvania Route 145 (PA 145) is a 20.89 mi long north–south state highway in the Lehigh Valley area of eastern Pennsylvania.', ' It connects Interstate 78 (I-78) and PA 309 in Lanark, Lehigh County to PA 248 in Weiders Crossing, Northampton County.', ' PA 145 is the main north-south arterial into Allentown, the third-largest city in the state.', ' The route enters the city on South 4th Street and follows multiple streets to downtown, where it follows the one-way pair of 6th Street northbound and 7th Street southbound.', ' North of Allentown in Whitehall Township, a seven-mile (11.2 km) portion of PA 145 is known as MacArthur Road, named in honor of General Douglas MacArthur.', ' MacArthur Road is a divided highway; between U.S. Route 22 (US 22) and Eberhart Road, it is six lanes wide with a Jersey barrier and jughandles while the remainder of the road a four-lane divided highway.', ' MacArthur Road is the location of the main commercial center of the Lehigh Valley.', ' North of Eagle Point, PA 145 becomes a two-lane undivided road that parallels the Lehigh River, crossing the river into Northampton County at Treichlers.', ' The route continues along the east bank of the river and passes through Walnutport before reaching its northern terminus.', ' PA 145 is dedicated as the Battle of the Bulge Veterans Memorial Highway in honor of the veterans who fought in the Battle of the Bulge.']], ['Periplus', ['A periplus ( ) is a manuscript document that lists the ports and coastal landmarks, in order and with approximate intervening distances, that the captain of a vessel could expect to find along a shore.', ' It served the same purpose as the later Roman itinerarium of road stops; however, the Greek navigators added various notes, which if they were professional geographers (as many were) became part of their own additions to Greek geography.', ' In that sense the periplus was a type of log.']], ['Consumer expectations test', ['In legal disputes regarding product liability, a consumer expectations test is used to determine whether the product is negligently manufactured or whether a warning on the product is defective.', ' Under this test, the product is considered defective if a reasonable consumer would find it defective.', ' As an example, a reasonable consumer might find exposed blades on a lawnmower, without plastic guards that could be installed for pennies, to be defective because the risk of not having the plastic guards is higher than the costs of installing those guards.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.598\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7cf29255429909bec768b8', 'answer': '150 million copies', 'question': 'The Ingerophrynus gollum is named after a character in a book that sold how many copies?', 'supporting_facts': [['Ingerophrynus gollum', 2], ['The Lord of the Rings', 2]], 'context': [['Suanfa tongzong', ['Suanfa tongzong (General Source of Computational Methods) is a mathematical text written by sixteenth century Chinese mathematician Cheng Dawei (1533–1606) and published in the year 1592.', ' The book contains 595 problems divided into 17 chapters.', ' The book is essentially general arithmetic for the abacus.', ' The book was the main source available to scholars concerning mathematics as it developed in China’s tradition.', ' Six years after the publication of Suanfa Tongzong, Cheng Dawei published another book titled \"Suanfa Zuanyao\" (\"A Compendium of calculating Methods\").', ' About 90% of the content of the new book came from the contents of four chapters of the first book with some rearrangement.', ' It is said that when Suanfa Tongzong was first published, it sold so many copies that the cost of paper went up and the lucrative sales resulted in unscrupulous people beginning to print pirated copies of the book with many errors.', ' it was this that forced the author to print an abridged version.']], ['Gollum', [\"Gollum is a fictional character from J. R. R. Tolkien's legendarium.\", ' He was introduced in the 1937 fantasy novel \"The Hobbit\", and became an important supporting character in its sequel, \"The Lord of the Rings\".', ' Gollum was a Stoor Hobbit of the River-folk, who lived near the Gladden Fields.', ' Originally known as Sméagol, he was corrupted by the One Ring and later named Gollum after his habit of making \"a horrible swallowing noise in his throat\".']], ['Ingerophrynus gollum', [\"Ingerophrynus gollum (Gollum's toad) is a species of true toad.\", ' It has only been recorded from Endau-Rompin National Park, Johor, in closed-canopy lowland forests in the early evening following brief periods of afternoon precipitation.', ' It is called \"gollum\" with reference of the eponymous character of The Lord of the Rings by J. R. R. Tolkien.']], ['The Lord of the Rings', ['The Lord of the Rings is an epic high fantasy novel written by English author and scholar J. R. R. Tolkien.', ' The story began as a sequel to Tolkien\\'s 1937 fantasy novel \"The Hobbit\", but eventually developed into a much larger work.', ' Written in stages between 1937 and 1949, \"The Lord of the Rings\" is one of the best-selling novels ever written, with over 150 million copies sold.']], ['Cosmological interpretation of quantum mechanics', ['The cosmological interpretation of quantum mechanics, proposed by Anthony Aguirre and Max Tegmark, is an interpretation of quantum mechanics that applies in the context of eternal cosmological inflation, which arguably predicts an infinite three-dimensional space with infinitely many planets and infinitely many copies of any quantum system.', ' According to this interpretation, the wavefunction for a quantum system describes not some imaginary ensemble of possibilities for what the system might be doing, but rather the actual spatial collection of identical copies of the system that exist in our infinite space.', ' Its collapse can be avoided.', ' Moreover, the quantum uncertainty that you experience simply reflects your inability to self-locate in space, i.e., to know which of your infinitely many copies throughout space is the one having your subjective perceptions.']], ['Gollum: How We Made Movie Magic', ['\"Gollum\" details how a three-week commission for Andy Serkis to provide a voice-over for Gollum grew into a five-year commitment to breathe life and soul into \"The Lord of the Rings\"\\' most challenging creation.', ' Andy Serkis tackles various subjects throughout the book, including character conception (Gollum\\'s \"cough\" is derived from his cat coughing up a hairball) as well as the hard work it took to act out Gollum and replace it with CGI.', ' He also discusses the controversy of whether he should have been eligible for an Academy Award for his work as Gollum.']], ['The Burning Red', ['The Burning Red is the third album by the American groove metal band Machine Head.', ' It is the band\\'s second best selling album in the US, selling as many copies in three years as \"Burn My Eyes\" sold in almost eight years (1994–2002) .', ' The album has sold over 134,000 copies in the US and it was certified silver in 2011 by the BPI for sales of 60,000 in the UK.']], ['Gollum (genus)', ['Gollum is a genus of ground sharks in the family Pseudotriakidae, native to the southwestern Pacific Ocean.', ' The genus was described in 1973 by biologist Leonard Compagno, who named it named after the character Gollum from J. R. R. Tolkien\\'s works, noting the species \"Gollum attenuatus\" (the slender smooth-hound) \"bears some resemblance in form and habits\".']], ['Ingerophrynus', ['Ingerophrynus is a genus of true toads with 12 species.', ' These species distributed in southern Yunnan and Indochina; peninsular Thailand and Malaya to Sumatra, Borneo, Java, Nias Island, Sulawesi, and the Philippines.', ' This genus was established after a major taxonomical revision of frogs.', ' Ten of the species in this genus were formerly considered species of the genus \"Bufo\".', ' In 2007 a new species, \"Ingerophrynus gollum\", was added to this genus.', ' This species is named after the character Gollum created by J. R. R. Tolkien.']], ['From Unknown Worlds', ['From Unknown Worlds is an anthology of fantasy fiction short stories edited by John W. Campbell, Jr. and illustrated by Edd Cartier, the first of a number of anthologies drawing their contents from the classic magazine \"Unknown\" of the 1930s-40s.', ' It was first published in magazine format by American company Street & Smith in 1948; the publication was an attempt to determine if there was a market for a revived \"Unknown\".', ' Street & Smith printed 300,000 copies, against the advice of John Campbell, but although it sold better than the original, too many copies were returned for the publisher to be willing to revive the magazine.', ' The first British edition was issued by Atlas Publishing in 1952; part of the run was issued in a hardcover binding.', ' This edition omitted the story \"One Man\\'s Harp.\"', '.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.599\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a807e6f5542995d8a8ddf80', 'answer': 'British Army', 'question': 'Who were the guns stolen from the HMS ship \"Marlin\" originally produced for?', 'supporting_facts': [['The Royal African Rifles', 0], ['Vickers machine gun', 0]], 'context': [['HSwMS Gustav V', ['HSwMS \"Gustav V\" was a \"Sverige\"-class coastal defence ship of the Swedish Navy.', ' The vessel was the third and last ship in the Sverige class along with HMS Sverige and HMS Drottning Victoria.', ' Gustav V was launched on September 15, 1917 at Kockums in Malmö and delivered to the Navy on January 9, 1922.', ' The design consisted of four 28\\xa0cm cannons and a secondary armament of eight 15.2\\xa0cm cannons.', ' During the interwar period, the ship underwent several modernizations and was one of the most powerful vessels in the fleet during the Second World War.', ' The ship was put in reserve in 1948, was decommissioned in 1957 and was later sold for scrapping in Karlskrona.', ' However, the ship remained at Berga Academy of War as of 1968.', \" Two of the ship's 15.2\\xa0cm guns are preserved in the battery at Häggmansberget in the defensive Kalix Line, around Kalix.\"]], ['Vickers machine gun', ['The Vickers machine gun or Vickers gun is a name primarily used to refer to the water-cooled .303 British (7.7 mm) machine gun produced by Vickers Limited, originally for the British Army.', ' The machine gun typically required a six to eight-man team to operate: one fired, one fed the ammunition, the rest helped to carry the weapon, its ammunition, and spare parts.', ' It was in service from before the First World War until the 1960s, with air-cooled versions of it on many Allied World War I fighter aircraft.']], ['HMS James Watt', ['HMS \"James Watt\" was a 91-gun steam and sail-powered second rate ship of the line.', ' She had originally been ordered as one of a two ship class, with her sister HMS\\xa0\"Cressy\" , under the name HMS \"Audacious\".', ' She was renamed on 18 November 1847 in honour of James Watt, the inventor of the steam engine.', ' She was the only Royal Navy ship to bear this name.', ' Both ships were reordered as screw propelled ships, \"James Watt\" in 1849, and \"Cressy\" in 1852.', ' \"James Watt\" became one of the four-ship \"Agamemnon\"-class of ships of the line.', ' They were initially planned as 80-gun ships, but the first two ships built to the design, HMS\\xa0\"Agamemnon\" and \"James Watt\", were rerated on 26 March 1851 to 91 guns ships, later followed by the remainder of the class.']], ['The Royal African Rifles', ['In August 1914 a consignment of Vickers machine guns are stolen off a Royal Navy ship, HMS \"Marlin\".', ' An RN Lieutenant aboard the ship goes undercover as a white hunter through British East Africa to find the weapons before they get into the hands of the Germans and alter the balance of power in Africa.']], ['History of the Sri Lanka Navy', ['In January, 1938 the Ceylon Naval Volunteer Force(CNVF) was created with Commander W.G. Beauchamp as Commanding Officer.', ' On 31 August 1939, the CNVF was mobilized for war duties.', ' Three years later, the CNVF was offered to, and accepted by the Royal Navy (RN) as a Volunteer Reserve, the \"Ceylon Royal Naval Volunteer Reserve\", or CRNVR.', ' It continued under RN operational and administrative command until March 1946.', ' With the end of the war, it reverted to Ceylon Government Control, though yet CRNVR in name.', ' In the 1939-1946 period, the CRNVR carried out several operational duties, mainly at sea.', ' Cutting its teeth on the Port Commission Tugs SAMSON and GOLIATH, it later manned and operated trawlers and Antarctic whalers converted as Minesweepers and fitted out with guns, submarine detection equipment and anti-submarine weaponry.', ' They were the HMS Overdale Wyke (the first ship to be purchased by the Government of Ceylon), HMS Okapi, HMS Semla, HMS Sambhur, HMS Hoxa, HMS Balta and HM Tugs Barnet and C 405.', ' In addition the CRNVR manned several Motor Fishing Vessels (MFV), Harbour Defence Motor Launch (HDML) and miscellaneous auxiliary vessels.', ' All were manned exclusively by CRNVR personnel.', ' These ships were meant to sweep and guard the approaches the harbors but were often used on extended missions outside Ceylon waters.', ' In the course of these operations, the ships came under enemy fire, recovered essential information from Japanese aircraft that were shot down, sailed to Akyab after the Burma front was opened in two FMVs for harbour duties and, was called upon to accept the surrender of the Italian Light Cruiser Eritrea and escort her to port with a prize crew on board.']], ['HMS Valiant (1863)', ['HMS \"Valiant\" was the second ship of the \"Hector\"-class armoured frigates ordered by the Royal Navy in 1861.', ' Her builders went bankrupt shortly after she was laid down, which significantly delayed her completion.', ' After being launched in 1863, she waited a further five years to receive her guns due to supply issues.', ' Upon being commissioned in 1868 the ship was assigned as the First Reserve guard ship for Southern Ireland, where she remained until she was decommissioned in 1885.', ' \"Valiant\" was hulked in 1897 as part of the stoker training school HMS \"Indus\" before becoming a storeship for kite balloons during the First World War.', ' The ship was converted to a floating oil tank in 1926 and served in that role until sold for scrap in 1956.']], ['Capture of the Vigilant', ['The Capture of the Vigilant involved Commodore Warren in HMS Superb (60 guns), Captain Durell in HMS Eltham (40 guns), Captain Calmady in HMS Launceston, Captain Douglas in HMS Mermaid and Captain John Rous of HMS Shirley Galley who fought the French ship Vigilant (64 guns) off Louisbourg.', ' Captain Douglas in the Mermaid (40 guns) engaged the French ship Vigilant.', ' John Rous in the Shirley Galley was the first to fire, giving the ship several broadsides into the stern.', ' Captain Durell was next to give a broadside.', ' The Commodore got alongside the ship they fired briskly, tearing the rigging and sails to pieces.', ' Fog settled in and the Vigilant got away.', ' In the morning, the Vigilant was visible and clearly wrecked.', ' They took 100 French sailors prisoner to Boston.']], ['HMS Unicorn (1748)', ['HMS \"Unicorn\" was a 28-gun \"Lyme\"-class sixth-rate frigate of the Royal Navy.', ' She was originally ordered as a 24-gun ship to the draft of the French privateer \"Tyger\".', ' The third vessel of the Royal Navy to bear the name, \"Unicorn\", as well as HMS\\xa0\"Lyme\" which was a near-sister, were the first true frigates built for the Royal Navy.', \" They were actually completed with 28 guns including the four smaller weapons on the quarterdeck, but the latter were not included in the ship's official establishment until 22 September 1756.\", ' The two ships differed in detail, \"Unicorn\" having a beakhead bow, a unicorn figurehead , two-light quarter galleries and only five pairs of quarterdeck gunports, while \"Lyme\" had a round bow, a lion figurehead, three-light quarter galleries and six pairs of quarterdeck gunports.']], ['HMS Hannibal (1896)', ['HMS \"Hannibal\" was a \"Majestic\"-class pre-dreadnought battleship built for the Royal Navy, and the sixth ship to bear the name HMS \"Hannibal\".', ' The ship was laid down at the Pembroke Dock in May 1894, she was launched in April 1896, and commissioned into the fleet in April 1898.', ' She was armed with a main battery of four 12 in guns and a secondary battery of twelve 6 in guns.', ' The ship had a top speed of 16 kn .']], ['HMS Benbow (1913)', ['HMS \"Benbow\" was the third of four \"Iron Duke\"-class battleship s of the Royal Navy, the third ship to be named in honour of Admiral John Benbow.', ' Ordered in the 1911 building programme, the ship was laid down at the William Beardmore and Company shipyard in May 1912, was launched in November 1913, and was completed in October 1914, shortly after the outbreak of the First World War.', ' The four \"Iron Duke\"s were very similar to the preceding \"King George V\"\\xa0class , with an improved secondary battery.', ' She was armed with a main battery of ten 13.5 in guns and twelve 6 in secondary guns.', ' The ship was capable of a top speed of 21.25 kn , and had a 12 in thick armoured belt.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.600\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7ce444554299683c1c63c1', 'answer': 'Andy García', 'question': 'Who played the crime boss considered the father of modern organized crime in the 1997 drama Hoodlum?', 'supporting_facts': [['Hoodlum (film)', 1], ['Lucky Luciano', 1]], 'context': [['St. John Cemetery (Queens)', ['St. John Cemetery is an official Roman Catholic burial ground located in Middle Village in the Queens borough of New York City.', ' It is one of nine official Roman Catholic burial grounds in the New York Metropolitan Area.', ' St. John, along with St. Charles/Resurrection Cemeteries in Farmingdale, Long Island, is among the largest cemeteries in New York State.', ' Since its opening, St. John has been the resting place of various famous and infamous people in New York City society, such as Mario Cuomo (1932–2015), Governor of the state of New York from 1983–1995, John F. Hylan (1868–1936), mayor of the city of New York from 1918–1925, Geraldine Ferraro (1935–2011), the first female Vice Presidential candidate representing a major American political party, Lucky Luciano (1897–1962), considered the father of modern organized crime in the United States, and John J. Gotti (1940–2002), the head of the New York City based Gambino crime family from 1985–2002.']], ['Johnny Jack Nounes', ['Johnny Jack Nounes, also known as the \"Beau Brummell of Galveston\", was an organized crime boss in Galveston, Texas, United States, during the early 1900s.', ' He, with one-armed George Musey, led the Downtown Gang, one of the two gangs which controlled most of the Galveston Crime Syndicate until the early 1930s.', ' They fought for control of the island against the rival Beach Gang led by Ollie Quinn and Dutch Voight.', ' As the prohibition era began, his gang came to be one of the dominant forces in the Galveston Crime Syndicate.', ' Galveston became the main port of entry for liquor supply in Texas and many parts of the Midwest.']], ['Frank Colacurcio', ['Francis Colacurcio, Sr. (June 18, 1917 – July 2, 2010) was an American businessman and boss of the Seattle crime family known for running strip clubs in Seattle, Washington.', ' He gained notoriety as a subject of ongoing federal investigations into organized crime in the city and was suspected of being an organized crime boss.']], ['Still at Large', ['Still at Large is the third and final album released by Crime Boss.', ' Following the release of his successful second album, \"Conflicts & Confusion\", Crime Boss left Suave House Records and formed his own independent label called Crime Lab Records and released \"Still at Large\" through it.', \" Though his previous two albums were produced by Suave House's T-Mix, Crime Boss himself handled a majority of the album's production. '\"]], ['James Marcello', ['James J. Marcello (born December 13, 1943), also known variously as Little Jimmy, Jimmy Light and as Jimmy the Man Marcello, is an imprisoned crime boss who was a front boss for the Chicago Outfit criminal organization in the 1980s, 1990s and early 2000s.', \" Organized crime observers identified Marcello as a figurehead during that period while the organization's day-to-day operations actually were run by John “No Nose” DiFronzo, Joseph “Joey the Clown” Lombardo, Joseph “Joe the Builder” Andriacchi and Angelo J. LaPietra.\", ' Marcello is the Cousin of Michael Hayes Marcello, and a cousin of Robert Urbinatti.', ' He is also the father of Rocco Marcello.']], ['Joseph Todaro Jr.', ['Joseph \"Big Joe\" Todaro Jr. (born 1945 or 1946) is a Buffalo, New York businessman and former organized crime figure involved in labor racketeering, loansharking, illegal gambling, narcotics, and murder for hire.', \" Joe Todaro Jr. became a business agent for the Laborers' International Union of North America (LIUNA) Local 210.\", ' In 1984, Joe Todaro Jr. allegedly became the underboss of the Buffalo crime family after his father became the new head of the crime family, upon the retirement of his predecessor Samuel \"Sam the Farmer\" Frangiamore.', \" In 1990, Joe Todaro Jr. resigned as business agent following investigations on the local's alleged ties to organized crime.\", ' Outside of organized crime, Todaro operates La Nova Pizzeria, a popular pizza restaurant in Buffalo.']], ['Ang Utol Kong Hoodlum', ['Ang Utol Kong Hoodlum (lit.', ' \"My Brother is a Hoodlum\") is a Filipino drama series developed for TV5 created by Deo J. Fajardo.', ' It stars JC de Vera and Jasmine Curtis-Smith.', ' It is a remake of the original movie where Robin Padilla played the role of Ben, a hoodlum, and Vina Morales, as Bing.', ' The movie was first released in 1991, then a sequel entitled \"Miss na Miss Kita: Ang Utol Kong Hoodlum 2\" was made the following year.', ' It is produced by Vic Del Rosario Jr., and Manuel V. Pangilinan and it also marks as the first primetime series produced by Viva Television for TV5 after a decade.']], ['Hoodlum (film)', ['Hoodlum is a 1997 American crime drama film that gives a fictionalized account of the gang war between the Italian/Jewish mafia alliance and the Black gangsters of Harlem that took place in the late 1920s and early 1930s.', ' The film concentrated on Ellsworth \"Bumpy\" Johnson (Laurence Fishburne), Dutch Schultz (Tim Roth), and Lucky Luciano (Andy García).']], ['Lucky Luciano', ['Charles \"Lucky\" Luciano ( ; born Salvatore Lucania November 24, 1897 – January 26, 1962) was an Italian-American mobster and crime boss.', ' Luciano is considered the father of modern organized crime in the United States for the establishment of the first Commission.', ' He was also the first official boss of the modern Genovese crime family.', ' He was, along with his associates, instrumental in the development of the National Crime Syndicate.']], ['Hoodlum (soundtrack)', ['Hoodlum is the soundtrack to the 1997 American crime drama film \"Hoodlum\".', ' It was released on August 12, 1997 by Interscope Records and consisted of a blend of hip hop and R&B music.', ' The soundtrack peaked at 94 on the \"Billboard\" 200 and 23 on the Top R&B Albums and contained Mobb Deep\\'s single \"Hoodlum\" which went to 29 on the Hot Rap Singles.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.601\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adcd6705542992c1e3a2426', 'answer': 'no', 'question': 'Are Der Rosenkavalier and I Capuleti e i Montecchi both comic operas?', 'supporting_facts': [['Der Rosenkavalier', 0], ['I Capuleti e i Montecchi', 0], ['I Capuleti e i Montecchi', 1]], 'context': [['Der Rosenkavalier', ['Der Rosenkavalier (The Knight of the Rose or The Rose-Bearer), Op. 59, is a comic opera in three acts by Richard Strauss to an original German libretto by Hugo von Hofmannsthal.', ' It is loosely adapted from the novel \"Les amours du chevalier de Faublas\" by Louvet de Couvrai and Molière’s comedy \"Monsieur de Pourceaugnac\".', ' It was first performed at the Königliches Opernhaus in Dresden on 26 January 1911 under the direction of Max Reinhardt, Ernst von Schuch conducting.', ' Until the premiere the working title was \"Ochs von Lerchenau\".', ' (The choice of the name Ochs is not accidental, for in German Ochs means ox, which depicts the character of the Baron throughout the opera.)']], ['Richard Strauss', ['Richard Georg Strauss (11 June 1864\\xa0– 8 September 1949) was a leading German composer of the late Romantic and early modern eras.', ' He is known for his operas, which include \"Der Rosenkavalier\", \"Elektra\", \"Die Frau ohne Schatten\" and \"Salome\"; his Lieder, especially his \"Four Last Songs\"; his tone poems, including \"Don Juan\", \"Death and Transfiguration\", \"Till Eulenspiegel\\'s Merry Pranks\", \"Also sprach Zarathustra\", \"Ein Heldenleben\", \"Symphonia Domestica\", and \"An Alpine Symphony\"; and other instrumental works such as \"Metamorphosen\" and his Oboe Concerto.', ' Strauss was also a prominent conductor in Western Europe and the Americas, enjoying quasi-celebrity status as his compositions became standards of orchestral and operatic repertoire.']], ['Lotte Lehmann', ['Charlotte \"Lotte\" Lehmann (February 27, 1888 – August 26, 1976) was a German soprano who was especially associated with German repertory.', ' She gave memorable performances in the operas of Richard Strauss, Richard Wagner, Ludwig van Beethoven, Puccini, Mozart, and Massenet.', ' The Marschallin in \"Der Rosenkavalier\", Sieglinde in \"Die Walküre\" and the title-role in \"Fidelio\" are considered her greatest roles.', ' During her long career, Lehmann also made more than five hundred recordings.', ' Her performances in the world of Lieder are considered among the best ever recorded.']], ['Luigi Scevola', ['Luigi Scevola (born Brescia, 1770 - died Milan, 1819) was an Italian dramatist.', ' He wrote in the style of Ugo Foscolo, and was the author of the tragedies \"Socrate\" (1804), \"Annibale in Bitinia\" (1806) and \"Saffo\" (1814).', ' One of his plays was the basis of the libretti for \"Giulietta e Romeo\" by Nicola Vaccai and \"I Capuleti e i Montecchi\" by Vincenzo Bellini.']], ['Karl Perron', ['Karl Perron, born Karl Pergamenter and also known as Carl Perron, (3 June 1858 – 15 July 1928) was a German bass-baritone.', ' A Kammersänger of the Dresden State Opera, he created leading roles in three operas by Richard Strauss – Jochanaan in \"Salome\", Orest in \"Elektra\", and Baron Ochs in \"Der Rosenkavalier\".']], ['Antonietta Marini-Rainieri', ['Antonietta Marini-Rainieri was an Italian operatic soprano active during the first half of the 19th century.', ' She was married to lauded operatic bass Ignazio Marini and often appeared on stage with him.', ' In 1835 she portrayed Giulietta opposite Amalia Schütz Oldosi as Romeo in the Teatro Regio di Parma\\'s first staging of Vincenzo Bellini\\'s \"I Capuleti e i Montecchi\".', ' At La Scala she portrayed roles in the world premieres of Giuseppe Verdi\\'s first two operas: Leonora in \"Oberto\" (1839) and the Marchesa del Poggio in \"Un giorno di regno\" (1840).', ' She also appeared at that house as the Princess of Navarra in the premiere of Gaetano Donizetti\\'s \"Gianni di Parigi\" (1839).', ' In 1843 she sang the title role in the premiere of Giovanni Pacini\\'s \"Maria, regina d\\'Inghilterra\" at the Teatro Carolino in Palermo.', ' She reprised that role in December 1843 at La Scala and at the Teatro Carlo Felice in February 1844.']], ['La sonnambula (Balanchine)', ['La sonnambula \"(The Sleepwalker)\" is a ballet by the co-founder and ballet master of New York City Ballet, George Balanchine, made to Vittorio Rieti\\'s music using themes from the operas of Vincenzo Bellini including \"La Sonnambula\", \"Norma\", \"I Puritani\" and \"I Capuleti e i Montecchi\" (1830–35) and with costumes by Karinska.', ' The premiere took place with the Ballet Russe de Monte Carlo on Wednesday, 27 February , at City Center of Music and Drama, New York; the City Ballet premiere was on 6 January 1965.']], ['Simon Gilbert (tenor)', ['Simon Gilbert (born 6 October 1937 in Hendon, London) is an English actor and tenor.', ' At the age of twenty one he took singing lessons with the teachers of the Australian soprano Joan Sutherland.', ' Gilbert gradually gravitated towards Opera and The Edinburgh Festival, where sang with Sutherland (in Haydn’s \"Orfeo\") and Luciano Pavarotti (in \"I Capuleti e i Montecchi\").', ' In 1967, he sang with Scottish Opera, for example appearing in \"L\\'anima del filosofo\".', ' Joining The Adelphi Theatre\\'s company for the musical \"Show Boat\", he played lead man to the show\\'s star, Cleo Laine.']], ['Margarethe Siems', ['Margarethe Siems (20 December 1879 – 13 April 1952) was a German operatic soprano and voice teacher.', ' A Kammersängerin of the Dresden State Opera, between 1909 and 1912 Siems created leading roles in three operas by Richard Strauss: Chrysothemis in \"Elektra\", the Marschallin in \"Der Rosenkavalier\", and Zerbinetta in \"Ariadne auf Naxos\".']], ['I Capuleti e i Montecchi', ['I Capuleti e i Montecchi (\"The Capulets and the Montagues\") is an Italian opera (\"Tragedia lirica\") in two acts by Vincenzo Bellini.', ' The libretto by Felice Romani was a reworking of the story of \"Romeo and Juliet\" for an opera by Nicola Vaccai called \"Giulietta e Romeo\" and based on the play of the same name by Luigi Scevola written in 1818, thus an Italian source rather than taken directly from William Shakespeare.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.602\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adf0ae45542992d7e9f9286', 'answer': 'Orissa', 'question': 'Where was the film set in which the daughter of Shashi Kapoor, along side Sarat Pujari and Navni Parihar, set?', 'supporting_facts': [['Aranyaka (film)', 0], ['Aranyaka (film)', 1], ['Aranyaka (film)', 2], ['Sanjana Kapoor', 0], ['Sanjana Kapoor', 1]], 'context': [['Aranyaka (film)', ['Aranyaka (A Trip Into the Jungle) is a 1994 Indian Hindi drama film directed by Apurba Kishore Bir.', ' The film stars Sarat Pujari, Navni Parihar, Sanjana Kapoor and Mohan Gokhale in lead roles.', ' Based on a short story \"Aranyaka\" by Manoj Das, the film is set in rural Orissa, where a formal local ruler organizes a hunt for his invited guests, which goes wrong.', ' The film highlights the clash between ruling class and indigenous people of the region.']], ['Dastaan (Zee)', ['Dastaan was a TV show that aired on Zee TV in mid 1990s.', ' The show was based in Dubai, stylishly shot, and starred Parmeet Sethi, Navni Parihar, Nishigandha Wad and Ashish Vidyarthi.']], ['Sarat Pujari', ['Sarat Pujari (8 August 1934 – 12 May 2014) was an Indian film actor, director and producer in Odia film industry (Ollywood).', ' He was originally from Jhaduapada, Sambalpur.']], ['Vakil Babu', ['Vakil Babu is a Hindi movie, which was released in April 1982.', ' The movie was produced by Jawahar Kapoor and P. K. Luthra and directed by Asit Sen.', ' The film stars Raj Kapoor alongside his younger brother Shashi Kapoor and also featuring Zeenat Aman, Rakesh Roshan, Kader Khan, Aruna Irani and Kishore Sahu.', \" This was Raj Kapoor's last leading film role and was also the first and only time he appeared onscreen with his brother Shashi, not counting Awara, wherein Shashi Kapoor appeared as a child actor.\"]], ['Jab Jab Phool Khile', [\"Jab Jab Phool Khile (Hindi: जब जब फूल खिले ; Urdu: \\u200e ; Translated: 'Whenever the flowers bloomed') is a 1965 Indian Hindi movie.\", ' It stars Shashi Kapoor and Nanda.', ' The story is of a poor boy who is a boatman in Kashmir and falls in love with a rich tourist.', ' The film became a \"blockbuster\" at the box office, was No. 2 in top ten grossing films at the Indian Box Office in 1965.', ' The songs by music composing duo Kalyanji Anandji assisted by then little-known Laxmikant Pyarelal are highlights of the film (lyrics by Anand Bakshi).', \" The film was screened in Algeria's cinema halls every two days for a couple of years; there was, in fact, public demand for it.\", ' Shashi Kapoor was one of the most successful Indian actors in North African countries like Algeria, Morocco and Libya.', ' In the souks of Marrakesh, even today some of the older shopkeepers will give you a discount if you are from the land of Shashi Kapoor.']], ['Dharitri', ['Dharitri is an Odia social satire drama released on 30 March 1973.', \" It is based on Amulya Kumari Patnaik's novel in the same name.\", ' Sarat Pujari, Prashant Nanda, Sriram Panda, Parbati Ghosh and Dhira Biswal acted in key roles.']], ['Prithvi Theatre', [\"Prithvi Theatre is one of Mumbai's best known theatres.\", ' It was built by Shashi Kapoor and his wife Jennifer Kapoor in memory of Prithviraj Kapoor, Shashi\\'s father, who had dreamt of having a \"home\" for his repertory theatre company, Prithvi Theatres, belongs to the Kapoor family, one of the most influential actor and director families in Bollywood.', \" Prithviraj Kapoor founded 'Prithvi Theatres', a travelling theatre company in 1944.\", ' The company ran for sixteen years.', ' Ved Segan was the architect who designed and built the Theatre with the supervision of Jennifer Kapoor.', ' Jennifer, (Trustee) supervised the building and running of the theatre until her death in 1984.', ' In 1978 Prithvi Theatre opened in Juhu, Mumbai.', ' Shashi Kapoor is the Managing Trustee, the daily affairs are looked after by Kunal Kapoor (Trustee) with a small but efficient team.', ' Prithvi Theatre has shows every day of the year (closed Mondays), hosts an annual Summertime programme of workshops and plays for children, the Memorial Concert on 28 February, an annual Theatre Festival in November, and many partnership programmes promoting language, poetry, international cinema & documentaries, performing arts – which are free to the public.']], ['Navni Parihar', ['Navni Parihar (born 22 March 1966) is an Indian film and television actress.', ' Navni Parihar acted in the film \"Rabba Main Kya Karoon\" in 2013 starring Arshad Warsi.', ' Navni is married to Animesh.', ' Navni is also playing an important role in \"Bani - Ishq Da Kalma\", which was earlier titled as \"Gurbani\".', ' Navni, who has worked two decades in Bollywood, also played the role of Indira Gandhi in Shekhar Kapoor documentary series \"Pradhanmantri\", and 7 RCR (TV Series).']], ['Sanjana Kapoor', ['Sanjna Kapoor (born 27 November 1967) is an Indian theatre personality and former Indian film actress of British and Indian descent.', ' She is the daughter of Shashi Kapoor and the late Jennifer Kendal.', ' She ran the Prithvi Theatre in Mumbai from 1993.', ' to February 2012.']], ['Shree Shree Mahalaxmi Puja', ['Shree Shree Mahalaxmi Puja (Odia: ଶ୍ରୀ ଶ୍ରୀ ମହାଲକ୍ଷ୍ମୀ ପୂଜା ) is a 1959 Indian Odia mythological film directed by Biswanath Nayak.', ' This is debut film of Sarat Pujari.', ' The tale of Goddess Laxmi leaving the temple to teach brothers Lord Jagannath and Lord Balabhadra, a lesson was presented in the movie.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.602\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7eec6f55429934daa2fc7c', 'answer': '69.7 million litres', 'question': 'The Hindmarsh Stadium is sponsored by a brewery that sold how many liters of beer in 2013 ?', 'supporting_facts': [['Hindmarsh Stadium', 0], ['Coopers Brewery', 2]], 'context': [['Coopers Brewery', ['Coopers Brewery Limited, the largest Australian-owned brewery, is based in the Adelaide suburb of Regency Park.', ' Coopers is known for making a variety of beers, the most famous of which are its Pale Ale and Sparkling Ale.', ' The brewery sold 69.7 million litres of beer in the 2013 financial year.', \" It is also the world's largest producer of homebrewing equipment.\", \" Its shares are primarily owned by the extended Cooper family, and the company's constitution and classes of shares makes it difficult to sell shares outside the family.\"]], ['Cervecería Nacional Dominicana', ['Cerveceria Nacional Dominicana (CND), is the primary beer producer in the Dominican Republic, the company is owned by AmBev and Grupo León Jimenes.', ' It was founded in 1929 by the American entrepreneur Charles H. Wanzer.', ' It was the first brewery in the Dominican Republic and the largest in the Antilles and Central America with sales of 3.8 million hectoliters.', ' It first released its major brand \"Presidente\" in 1935, and has since expanded to other brands such as Bohemia Especial, Presidente Light and Ambar.', \" The first two are pilsener beers that fall in the category of lager beers, and the latter is the company's first incursion into dark beer.\", ' CND also distributes Miller products and Heineken.', ' Its current brewery complex was opened in 1951.', ' It employs 2,500 people and produces up to 500 million liters of beer.']], ['Hindmarsh Stadium', ['The Hindmarsh Stadium (currently known as the Coopers Stadium due to sponsorship from the Adelaide-based Coopers Brewery) is a multi-purpose stadium located in Adelaide, South Australia.', ' It is the home of the Australian A-League team, Adelaide United.']], ['2008 AFC Champions League Final', ['The 2008 AFC Champions League Final was a two-legged football tie to determine the 2008 champions of Asian club football.', ' Gamba Osaka defeated Adelaide United 5-0 on aggregate to take the title.', \" The first leg took place on 5 November 2008 at 19:00 local time (UTC+9) at Osaka Expo '70 Stadium in Osaka and the second leg took place on 12 November 2008 at 19:30 local time () at Hindmarsh Stadium, Adelaide.\"]], ['Haandbryggeriet', ['Haandbryggeriet is a Norwegian brewery founded in 2005 by Jens Maudal, Rune Eriksen, Arne Eide and Egil Hilde.', ' The brewery was situated at the site of an old textile factory in Drammen, then in a railroad yard, and now resides in an old industrial building.', ' Their brewing equipment was bought used in England and has a capacity of about 900 liters per batch.', ' Production in 2006 was near 40,000 liters.', ' In 2012, production was expected to be approximately 350,000 liters, using a ,800 liter brewing equipment.', ' In 2013, they upgraded yet again, to a 5,000 liter brewing tank.']], ['Birra Tirana', ['Birra Tirana \"(English: Tirana Beer )\" is a beer company based in Tirana, Albania.', ' It is the largest beer producer and the largest selling beer in the country.', ' It is also exported and sold in Kosovo and the United States.', ' The company is fabricated by Birra Malto Brewery.', ' It currently produces three different beer brands.', ' Birra Tirana is sold both in bottles and cans of 0.33 lit and 0.5 lit and also in kegs 30 liters and 50 liters.']], ['1995 National Soccer League Grand Final', ['The 1995 National Soccer League Grand Final was the championship match of the 1994–95 National Soccer League season and was played between Adelaide City and Melbourne Knights at Hindmarsh Stadium on 7 May 1995.']], ['Adelaide United FC', ['Adelaide United Football Club is a professional soccer club based in Adelaide, South Australia, Australia.', ' The club participates in the A-League under licence from Football Federation Australia.', ' The club was founded in 2003 to fill the place vacated by Adelaide City and West Adelaide in the former National Soccer League (NSL), and is now the sole team from the state of South Australia in the A-League.', \" Adelaide United's home ground is Hindmarsh Stadium.\", ' Adelaide United were premiers in the inaugural 2005–06 A-League season, finishing 7 points clear of the rest of the competition, before finishing third in the finals.', ' They were Premiers again in 2015/16 finishing just one point ahead of second place Western Sydney.', ' The Reds made the Grand Finals of the 2006–07, 2008–09 and 2015–16 seasons, losing the on the first two occasions to Melbourne Victory.']], ['West Adelaide SC', ['West Adelaide Soccer Club is an Australian soccer club from Adelaide, Australia currently playing in the National Premier Leagues South Australia.', ' It participated in the National Soccer League from the 1977 season until the end of the 1998/99 season, except for the periods 1987–89 and 1990–91.', ' It was also known as West Adelaide Hellas and Adelaide Sharks.', ' They played in various blue and white strips, and played most of their NSL home games at Hindmarsh Stadium.', ' The senior arm of the club re-formed in 2008 and is the FFSA National Premier League 2015 Premiers and 2015 Champions , coached by one of the former NSL players of the club, Paul Pezos.']], ['Beer in Russia', ['In Russia, beer (Russian: пиво \"pivo\") is the second most popular alcoholic drink after vodka, seen by many as a less harmful alternative.', ' The average Russian person drank about 12.5 liters of pure alcohol in 2010, with vodka accounting for more than five liters and beer about four liters.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.603\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7625e8554299109176e66a', 'answer': '1865', 'question': 'In which year was the King who made the 1925 Birthday Honours born?', 'supporting_facts': [['1925 Birthday Honours', 0], ['George V', 0]], 'context': [['2017 Birthday Honours', [\"The 2017 Queen's Birthday Honours are appointments by some of the 16 Commonwealth realms of Queen Elizabeth II to various orders and honours to reward and highlight good works by citizens of those countries.\", \" The Birthday Honours are awarded as part of the Queen's Official Birthday celebrations during the month of June.\", \" The Queen's Birthday Honours for the United Kingdom were announced on 16 June; the honours for New Zealand were announced on 5 June and for Australia on 12 June.\"]], ['2014 Birthday Honours', ['The 2014 Birthday Honours were appointments by some of the 16 Commonwealth realms of Queen Elizabeth II to various orders and honours to reward and highlight good works by citizens of those countries.', \" The Birthday Honours are awarded as part of the Queen's Official Birthday celebrations during the month of June.\", \" The Queen's Birthday Honours were announced on 14 June 2014 in the United Kingdom, on 9 June 2014 in Australia, on 2 June 2014 in New Zealand, on 14 June 2014 in Grenada, Papua New Guinea, Solomon Islands, Tuvalu, Saint Lucia and Belize.\"]], ['2015 Birthday Honours', [\"The 2015 Queen's Birthday Honours are appointments by some of the 16 Commonwealth realms of Queen Elizabeth II to various orders and honours to reward and highlight good works by citizens of those countries.\", \" The Birthday Honours are awarded as part of the Queen's Official Birthday celebrations during the month of June.\", \" The Queen's Birthday Honours were announced on 1 June 2015 in New Zealand, on 8 June in Australia, and on 12 June in the United Kingdom, in Grenada, Papua New Guinea, Solomon Islands, Tuvalu, Saint Lucia and Belize.\"]], ['2016 Birthday Honours', [\"The 2016 Queen's Birthday Honours are appointments by some of the 16 Commonwealth realms of Queen Elizabeth II to various orders and honours to reward and highlight good works by citizens of those countries.\", \" The Birthday Honours are awarded as part of the Queen's Official Birthday celebrations during the month of June.\", \" The Queen's Birthday Honours were announced on 6 June 2016 in New Zealand and 10 June in the United Kingdom.\"]], ['Birthday Honours', [\"King's/Queen's Birthday Honours is, in some Commonwealth realms, the marking of the reigning monarch's official birthday by granting various individuals appointment into national or dynastic orders or the award of decorations and medals.\", ' The honours are presented by the monarch or a viceregal representative.', ' New Year Honours and Birthday Honours are bestowed each year.', ' All royal honours are published in the relevant gazette, and the daily newspapers of each realm.']], ['1965 Birthday Honours', [\"The Queen's Birthday Honours 1965 were appointments in many of the Commonwealth realms of Queen Elizabeth II to various orders and honours to reward and highlight good works by citizens of those countries.\", ' The appointments were made to celebrate the official birthday of The Queen.', ' The announcement date varies from year to year.', \" The 1965 Queen's Birthday Honours were announced on 12 June for the United Kingdom, Australia, New Zealand, Sierra Leone, Jamaica, Trinidad and Tobago, Malawi, and the Gambia.\"]], ['George V', ['George V (George Frederick Ernest Albert; 3 June 1865 – 20 January 1936) was King of the United Kingdom and the British Dominions, and Emperor of India, from 6 May 1910 until his death in 1936.']], ['1951 Birthday Honours', [\"The King's Birthday Honours 1951 were appointments in many of the Commonwealth realms of King George VI to various orders and honours to reward and highlight good works by citizens of those countries.\", ' The appointments were made to celebrate the official birthday of the King, and were published on 1 June 1951 for the British Empire, Australia, New Zealand, Ceylon, and Pakistan.', ' These were the last Birthday Honours awarded by George VI, who died eight months later.']], ['2013 Birthday Honours', ['The 2013 Birthday Honours were appointments by some of the 16 Commonwealth realms of Queen Elizabeth II to various orders and honours to reward and highlight good works by citizens of those countries.', \" The Birthday Honours are awarded as part of the Queen's Official Birthday celebrations during the month of June.\", \" The Queen's Birthday Honours were announced on 15 June 2013 in the United Kingdom, on 10 June 2013 in Australia on 3 June 2013 in New Zealand, on 15 June 2013 in Grenada, Papua New Guinea, Solomon Islands, Tuvalu, Saint Lucia and Belize.\"]], ['1925 Birthday Honours', ['The 1925 Birthday Honours were appointments by King George V to various orders and honours to reward and highlight good works by citizens of the British Empire.', ' The appointments were made to celebrate the official birthday of The King, and were published in \"The London Gazette\" on 3 June 1925.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.605\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac562115542993e66e823bf', 'answer': 'Taiwanese Hokkien, Cantonese, Japanese, Indonesian and English', 'question': 'The moon represents my heart is a mandarin song made famous by Teresa Teng, what other language does she sing in?', 'supporting_facts': [['The Moon Represents My Heart', 0], ['The Moon Represents My Heart', 1], ['Teresa Teng', 3]], 'context': [['Tian mi mi', ['Tian Mi Mi (; literally \"very sweet\") is a 1979 Mandarin Chinese song by Teresa Teng.', ' The lyrics were written by Zhuang Nu (莊奴, 1922–2016)Chinese Music - Page 126 Jie Jin - 2011 -\"The songs of Teresa Teng, including Story of Little Town (Xiao Cheng Gu Shi), Sweetness (Tian Mi Mi) and I only Care about You (Wo Zhi Zai Hu Ni), were widely spread on the mainland. \"', '</ref> The film is named after and features the Teresa Teng song.']], ['Green Island Serenade', ['\"Green Island Serenade\" (; also known as \"Serenade of Green Island\") is a Mandarin Chinese classic song composed in 1954 by Zhou Lanping (周藍萍), first performed by Zi Wei (紫薇).', ' Although many people believed that Teresa Teng had made this song famous, there is no evidence that Teresa Teng had sung this song.', ' The lyrics of this song were probably written by either Pan Yingjie (潘英傑) or Gao Yudang.']], ['Teresa Teng Memorial Hall', ['The Teresa Teng Memorial Hall () is a memorial hall in Gushan District, Kaohsiung, Taiwan dedicated to singer Teresa Teng.']], ['The Moon Represents My Heart', ['\"The Moon Represents My Heart\" () is a Mandarin song.', ' It was made famous by Teresa Teng.']], ['A Tribute to Teresa Teng', [\"A Tribute to Teresa Teng - A Rocking Farewell (告别的摇滚) is a May 1995 tribute album by many of the leading lights of Beijing's first generation rock bands.\", \" Although the take of hardcore rock acts like Zang Tianshuo and 1989 on the gentle songs of Teresa Teng is rough edged, like Faye Wong's tribute Decadent Sound of Faye (菲靡靡之音) which appeared two months later, the album was a genuine and sincere tribute to the childhood appreciation of Teng as one of the first pop singers to be heard in China.\"]], ['Dandan youqing', ['Dàndàn yōuqíng () is a 1983 Mandarin Chinese album by Teresa Teng, first distributed by Polydor Records, Ltd. (also called Polygram now owned by Universal Music Group), from Hong Kong and Kolin Records (歌林) from Taiwan.', ' It contains twelve songs, which use poems from the Tang and Song Dynasties as lyrics.']], ['I Only Care About You', ['\"I Only Care About You\" () is a Mandarin song by Taiwanese singer Teresa Teng.']], ['Love Love Love (Linda Chung album)', ['Love Love Love is the fourth album by Linda Chung, and was released on 13 November 2012.', ' It contains 11 tracks, of which 3 are Mandarin while the rest are Cantonese.', ' \"The Moon Represents My Heart\" is a remake of a famous song from the artist Teresa Teng.', \" In an interview, Chung expressed that the album's main theme was the idea of happiness.\", ' Whilst at a promotional event for lovelovelove, Linda also stated that she has now saved enough songs to hold a concert.']], ['Teresa Teng', ['Teresa Teng (29 January 1953 – 8 May 1995) was a Taiwanese singer.', ' She was known for her folk songs and romantic ballads, such as \"When Will You Return?', '\" and \"The Moon Represents My Heart\".', ' She recorded songs not only in her native Mandarin but also in Taiwanese Hokkien, Cantonese, Japanese, Indonesian and English.']], ['When Will You Return?', ['\"When Will You Return?\"', ' () is a Chinese song first sung by Zhou Xuan in 1937, but now better known as a song by Teresa Teng.', ' The song has also been variously translated as \"When Will the Gentleman Come Back Again?\"', ' or \"When Will You Come Back Again?\"', \" The lyrics were written by Huang Jiamo (黄嘉謨 ) to a tune composed by Liu Xue'an (劉雪庵 ).\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.605\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5add352b5542990dbb2f7dc4', 'answer': 'Frank Sinatra, Dean Martin, Sammy Davis Jr.', 'question': 'Who were the lead members of the group of actors who were in the film which was remade as the first film of \"Ocean\\'s Trilogy\"?', 'supporting_facts': [[\"Ocean's Thirteen\", 0], [\"Ocean's Thirteen\", 1], ['Rat Pack', 0], ['Rat Pack', 1], ['Rat Pack', 2]], 'context': [['Bad Animals', ['Bad Animals is the ninth studio album by the American rock band Heart, released on June 6, 1987 by Capitol Records.', \" The album continues the mainstream hard rock style from the band's 1985 self-titled release, all while enjoying similar success.\", ' RIAA certification as of 1992 is three times platinum, indicating sales in excess of three million copies in the US alone.', ' The album reached No. 2 on the U.S. \"Billboard\" 200 chart in August 1987.', ' \"Bad Animals\" contained the number one hit single \"Alone\", while \"Who Will You Run To\" reached No. 7, \"There\\'s the Girl\" reached No. 12, and \"I Want You So Bad\" reached No. 49.', ' The album is notable for containing only a few tracks with writing contributions from lead members Ann and Nancy Wilson.', ' It received a Grammy nomination for Best Rock Performance by a Duo or Group with Vocal.']], ['Khaidi', ['Khaidi (English: \"Prisoner\") is a 1983 Indian Telugu-language action film starring Chiranjeevi and directed by A. Kodandarami Reddy.', ' Madhavi was the heroine.', \" It became a major blockbuster and a turning point in Chiranjeevi's career winning unlimited fan following.\", ' It was remade into Kannada with Vishnuvardan with the same name.', ' The film is loosely based on the 1982 film \"First Blood\".', ' Later, this movie has brought big name to Chiranjeevi and he made Khaidi No. 786 as 100th movie and his 150th movie is Khaidi No. 150.', ' It is thus the first film in Chiranjeevi\\'s highly successful \"Khaidi\" trilogy.']], ['Antahpuram', ['Antahpuram (English: \"Palace\") is a 1998 Telugu action drama film produced by P.Kiran on Anandi Art Creations banner, directed by Krishna Vamsi.', ' The film stars Jagapati Babu, Soundarya, Prakash Raj, and Sai Kumar in prominent roles with music composed by Ilaiyaraaja.', ' Prakash Raj won National Film Award\\xa0– Special Jury Award / Special Mention (Feature Film) for his performance in the film.', ' The film won three Filmfare Awards South for Best Film, Best Director and Best Actress.', ' The film also went on to win the Nandi Award for Best Supporting Actor for Jagapati Babu, Nandi Award for Best Female Playback Singer for S. Janaki and Nandi Award for Best Female Dubbing Artist for Saritha.', ' This film is famous for being the first film in Telugu to have Rayalaseema faction as its main plot.', ' The film was later remade in 1999 in Tamil with the same name with Parthiban replacing Jagapathi Babu and it was remade in Hindi as \"\" (2003).']], ['Chitram', ['Chitram is a 2000 Telugu film written and directed by Teja.', ' This film stars Uday Kiran & Reema Sen in the lead roles and marked the debut of the lead actors.', ' The film was sensational hit in 2000.', ' The film is produced by Ramoji Rao and has music composed by R.P. Patnaik.', ' The film was released as \"Chithiram\" in Tamil in 2001 with the film mostly featuring scenes dubbed from the original although scenes involving Manivannan and Senthil were added.', ' \"Chitram\" and \"Nuvve Kavali\" were the two movies which set a trend of college campus romance stories in the same year.', ' The movie was remade in Kannada as \"Chithra\", starring Prabhu Deva\\'s brother Nagendra Prasad and Rekha Vedavyas in 2001 under the direction of Dinesh Baboo and the production of Ramoji Rao.', ' The movie was a superhit in Kannada and marked the debut of the lead actors.']], ['Khilona (1970 film)', ['Khilona (English: Toy ) is a 1970 Indian drama film, produced by L.V. Prasad on Prasad Productions Pvt.Ltd.', ' banner and directed by Chander Vohra.', ' Starring Sanjeev Kumar, Mumtaz, Jeetendra in lead roles.', ' Other actors in supporting roles are Shatrughan Sinha, Durga Khote, Ramesh Deo, Jagdeep and music composed by Laxmikant-Pyarelal.', ' The film recorded as \"Super Hit\" at the box office.', ' The film was a remake of Telugu film \"Punarjanma\".', ' The film was not remade in Tamil instead both Hindi and Tamil version were simultaneously made.', ' Tamil version had \"Engirundho Vandhaal\" with Sivaji Ganesan and Jayalalithaa.', ' The film was also remade in Malayalam as \"Amrithavaahini\".']], ['The Housemaid (1960 film)', ['The Housemaid (하녀, \"Hanyeo\") is a 1960 black-and-white South Korean film.', ' It was directed by Kim Ki-young and starred Lee Eun-shim, Ju Jeung-nyeo and Kim Jin-kyu.', ' It has been described in Koreanfilm.org as a \"consensus pick as one of the top three Korean films of all time\".', ' This was the first film in Kim\\'s \"Housemaid\" trilogy followed by \"Woman of Fire\".', ' The film was remade in 2010 by director Im Sang-soo.']], [\"Ocean's Thirteen\", [\"Ocean's Thirteen is a 2007 American comedy heist film directed by Steven Soderbergh and starring an ensemble cast.\", ' It is the third and final film in the Soderbergh-directed \"Ocean\\'s Trilogy\", following the 2004 sequel \"Ocean\\'s Twelve\" and the 2001 film \"Ocean\\'s Eleven\", which itself was a remake of the 1960 Rat Pack film \"Ocean\\'s 11\".', ' All the male cast members reprise their roles from the previous installments, but neither Julia Roberts nor Catherine Zeta-Jones returns.']], ['Marc Barthel', [\"Marc Barthel (also known as Jesse D'Lane) is a singer, actor, songwriter and dubbing actor from Berlin, Germany.\", \" He was born on Oct. 04, 1989 in Berlin and first became famous as one of the lead members of the pop group ''.\", \" He left the group in 2007 to focus on his solo music career as 'Jesse D'Lane'.\", ' Furthermore, he has since then been working as an actor in television, film and commercial productions.']], ['Higher Education Recruitment Consortium', ['The first HERC was established in Northern California in 2000 with Stanford, the University of California at Berkeley, and the University of California at Santa Cruz as lead members, with the goal of allowing colleges and universities to collaborate on the recruitment of faculty, staff, and executives.', ' There are currently eleven regional HERCs in the United States, consisting of over 550 campuses in 22 states and the District of Columbia.']], ['Rat Pack', ['The Rat Pack is a term used by the media to refer to an informal group of entertainers centered on the Las Vegas casino scene.', ' Having its origins in a group of friends that met at the Los Angeles home of Humphrey Bogart and Lauren Bacall, by the 1960s, it was the name used by the press and the general public to refer to a later variation of the group that called itself \"the Summit\" or \"the Clan\", featuring Frank Sinatra, Dean Martin, Sammy Davis Jr., Peter Lawford and Joey Bishop among others; they appeared together on stage and in films in the early 1960s, including the movies \"Ocean\\'s 11\", \"Sergeants 3\", and \"Robin and the 7 Hoods\" (in the last film, Bing Crosby replaced Lawford).', \" Sinatra, Martin, and Davis were regarded as the group's lead members.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.605\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a77aa095542995d83181260', 'answer': 'Pratt & Whitney F100', 'question': 'Which aircraft engine powers a General Dynamics aircraft of which over 4500 have been built?', 'supporting_facts': [['Pratt &amp; Whitney F100', 0], ['General Dynamics F-16 Fighting Falcon', 0], ['General Dynamics F-16 Fighting Falcon', 2]], 'context': [['General Dynamics Mission Systems', ['General Dynamics Mission Systems is a business unit of American defense and aerospace company General Dynamics.', ' General Dynamics Mission Systems integrates secure communication and information systems and technology.', ' General Dynamics Mission Systems has core manufacturing in secure communications networks; radios and satellite technology for the defense, cyber, public safety, and intelligence communities.']], ['Curtiss OX-5', ['The Curtiss OX-5 was an early V-8 American liquid-cooled aircraft engine built by Curtiss.', ' It was the first American-designed aircraft engine to enter mass production, although it was considered obsolete when it did so in 1917.', ' It nevertheless found widespread use on a number of aircraft, perhaps the most famous being the JN-4 \"Jenny\".', ' Some 12,600 units were built through early 1919.', ' The wide availability of the engine in the surplus market made it common until the 1930s, although it was considered unreliable for most of its service life.', ' Today, the engine can be found powering many Edwardian automobile racing specials on the historic racing scene.']], ['Pratt &amp; Whitney F100', ['The Pratt & Whitney F100 (company designation JTF22) is an afterburning turbofan engine manufactured by Pratt & Whitney which powers the F-15 Eagle and F-16 Fighting Falcon.']], ['General Dynamics F-16 Fighting Falcon', ['The General Dynamics F-16 Fighting Falcon is a single-engine supersonic multirole fighter aircraft originally developed by General Dynamics (now Lockheed Martin) for the United States Air Force (USAF).', ' Designed as an air superiority day fighter, it evolved into a successful all-weather multirole aircraft.', ' Over 4,500 aircraft have been built since production was approved in 1976.', ' Although no longer being purchased by the U.S. Air Force, improved versions are still being built for export customers.', ' In 1993, General Dynamics sold its aircraft manufacturing business to the Lockheed Corporation, which in turn became part of Lockheed Martin after a 1995 merger with Martin Marietta.']], ['General Dynamics F-111C', ['The General Dynamics F-111C (nicknamed \"Pig\") is a variant of the F-111 Aardvark medium-range interdictor and tactical strike aircraft, developed by General Dynamics to meet Australian requirements.', ' The design was based on the F-111A model but included longer wings and strengthened undercarriage.', ' The Australian government ordered 24 F-111Cs to equip the Royal Australian Air Force (RAAF) in 1963, but the aircraft were not delivered until 1973 because of long-running technical problems.', ' During 1979 and 1980 four of these aircraft were converted to the RF-111C reconnaissance variant.', ' Four ex-United States Air Force (USAF) F-111As were purchased by Australia and converted to F-111C standard in 1982 to replace F-111Cs destroyed during accidents.', ' Australia also operated 15 F-111Gs between 1993 and 2007, mainly for conversion training.', ' The RAAF retired its remaining F-111Cs in December 2010.', ' In Australian military and aviation circles, the F-111 Aardvark was affectionately known as the \"Pig\", due to its long snout and terrain-following ability.']], ['Gnome Omega', ['The Gnome 7 Omega (commonly called the Gnome 50\\xa0hp) is a French seven-cylinder, air-cooled aero engine produced by Gnome et Rhône.', ' It was shown at the Paris Aero Salon held in December 1908 and was first flown in 1909.', \" It was the world's first aviation rotary engine produced in quantity.\", ' Its introduction revolutionized the aviation industry and it was used by many early aircraft.', ' It produced 50\\xa0horsepower (37\\xa0kW) from its capacity of 8\\xa0litres (488\\xa0cubic inches).', ' A Gnome Omega engine powers the 1912 Blackburn Monoplane, owned and operated by the Shuttleworth Collection, the oldest known airworthy British-designed aeroplane worldwide.', ' A two-row version of the same engine was also produced, known as the Gnome 14 Omega-Omega or Gnome 100\\xa0hp.', \" The prototype Omega engine still exists, and is on display at the United States' National Air and Space Museum.\"]], ['General Dynamics F-111K', ['The General Dynamics F-111K was a planned variant of the General Dynamics F-111 Aardvark medium-range interdictor and tactical strike aircraft by General Dynamics, to meet a requirement for such an aircraft for the Royal Air Force.']], ['Robert H. Widmer', ['Robert Henry Widmer (May 17, 1916 – June 20, 2011) was an American aeronautical engineer who specialized in designing aircraft for the military.', ' He spent his career working for Convair which became General Dynamics, then Lockheed, and then Lockheed Martin.', ' His feisty personality and at times insubordinate attitude at one time led company leaders to strongly consider firing him.', ' However, his brilliance at envisioning and designing desirable aircraft years before there was even a market for them led to his appointment as Vice President for science and engineering for all of General Dynamics.']], ['Martin/General Dynamics RB-57F Canberra', ['The Martin/General Dynamics RB-57F Canberra is a specialized strategic reconnaissance aircraft developed in the 1960s for the United States Air Force by General Dynamics from the Martin B-57 Canberra tactical bomber, which itself was a license-built version of the English Electric Canberra.', ' It was operationally assigned to the Air Weather Service for weather reconnaissance involving high-altitude atmospheric sampling and radiation detection in support of nuclear test monitoring, but four of the 21 modified aircraft performed solely as strategic reconnaissance platforms in Japan and Germany.']], ['Canaero Toucan', ['The Canaero Toucan is a Canadian high-wing, two seats in tandem, twin engine push-pull configuration, twin-boom ultralight kit aircraft that was produced from 1983 to the late 1980s by Canaero Dynamics Aircraft of Rexdale, Ontario.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.606\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac00cde554299012d1db58a', 'answer': 'no', 'question': 'Are both movies Nerdcore Rising and What Would Jesus Buy? focused on similar topics?', 'supporting_facts': [['Nerdcore Rising (film)', 0], ['What Would Jesus Buy?', 0], ['What Would Jesus Buy?', 1], ['What Would Jesus Buy?', 2]], 'context': [['Big Tent Revival', ['Big Tent Revival is a Christian rock band that formed in 1991, toured extensively, disbanded in 2000, and reformed in 2012.', ' They were featured at the Harvest Crusades.', ' Their most popular songs were \"Two Sets of Jones\\'\", \"Choose Life\", and \"What Would Jesus Do?\"', '.', \" The first told a story about two different couples in which one trusted in Jesus through the storms of life and the other didn't.\", ' The second was used as an invitational at Harvest Crusade altar calls.', ' The last was part of the WWJD movement that encouraged people to consider what Jesus would do in real-life situations.']], ['Agriculture Reform, Food, and Jobs Act of 2013', ['The Agriculture Reform, Food, and Jobs Act of 2013 ( ), also commonly referred to as \"the farm bill,\" is one of two United States \"farm bills\" that were introduced in the 113th United States Congress.', ' The Agriculture Reform, Food, and Jobs Act of 2013 is the bill that was introduced into the United States Senate.', ' A second bill, the Federal Agriculture Reform and Risk Management Act of 2013 ( ) was introduced into the United States House of Representatives.', ' The two bills cover similar topics and programs, but have significantly different provisions.', ' The Agriculture Reform, Food, and Jobs Act of 2013 passed the Senate on June 10, 2013 and has received the support of the President.']], ['Nerdcore Rising', ['Nerdcore Rising is the official debut album by nerdcore rapper MC Frontalot.', ' The album was first released on August 27, 2005 at the Penny Arcade Expo.']], ['Everyman (TV series)', ['Everyman is a British television documentary series that aired on BBC One in a late-night slot on Sunday evenings between 1977 and 2005.', ' Its subject matter tended to be focused on moral and religious issues, often in the form of a film in which individuals would discuss their thoughts.', ' One edition from 1990, \"A Game of Soldiers\" concerned a group of soldiers exploring their feelings about being trained to kill.', ' Throughout much of its time on air, series of \"Everyman\" aired alternately with \"Heart of the Matter\", a debate series which featured somewhat similar topics.', ' Both series were cancelled in the 2000s after the BBC revamped the output of its religious programming.']], ['What Would Jesus Buy?', ['What Would Jesus Buy?', ' is a 2007 documentary film produced by Morgan Spurlock and directed by Rob VanAlkemade.', ' The title is a take-off on the phrase \"What would Jesus do?', '\".', ' The film debuted on the festival circuit on March 11, 2007, at the South By Southwest (SXSW) conference in Austin, Texas.', ' It went into general U.S. release on November 16, 2007.']], ['Nerdcore Rising (film)', ['Nerdcore Rising is a documentary/concert film starring MC Frontalot and other nerdcore hip hop artists such as mc chris, Wheelie Cyberman of Optimus Rhyme and MC Lars, with contributors from artists such as \"Weird Al\" Yankovic, Prince Paul, and Brian Posehn.']], ['The Secret Rulers of the World', ['The Secret Rulers of the World is a five-part documentary series, produced by World of Wonder Productions and written, directed by, and featuring Jon Ronson.', ' The series was first shown on Channel 4 in April and May 2001.', \" The series details Ronson's encounters with conspiracy theorists.\", ' It accompanies Ronson\\'s book \"\", which covers similar topics and describes many of the same events.']], ['Clean Head', ['Clean Head is Oceana\\'s first EP, and a follow-up to their second release \"Birth.Eater\".', ' The album was written with the intent of being the B-sides to \"Birth.Eater\" and covers similar topics.', ' The record focuses primarily around the idea of finding beauty in life through whatever way you see fit.', ' This album shows a vast sound change and maturity of the band as a whole and was very well received by fans and critics.', ' Clean Head will be released as a Hot Topic exclusive, and on various online distribution services.', ' This new EP has a total of four songs.', ' The album was released on May 11, 2010. \"', 'Birth.Eater\" will also be re-released by Distort Entertainment, with the four new EP tracks.', ' This is also the last release to feature guitarist Jack Burns as well as the last to feature the \"Oceana\" name.']], ['Nerdcore Hiphop (album)', ['Nerdcore Hiphop is a demo album by MC Frontalot, which first gave a name to the nerdcore hip hop genre, as well as the name of a song on that album.', ' Because it was only released via the internet, the track listing is unordered, and includes tracks released from 1999 until his first commercial album, \"Nerdcore Rising\", in 2005.', ' The songs are therefore listed here in the order of which they were released.', ' All listed songs are freely available for download through his website along with several remixes, mostly by Song Fight!', ' regulars.']], ['Discovery Real Time France', ['Discovery Real Time was a French television channel broadcasting lifestyle programmes about decorating, fashion, cooking and similar topics.', ' It primarily targeted women.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.607\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a83697e5542996488c2e441', 'answer': 'May 22', 'question': ' Park Seo-joon is best known for his role in a series that premiered on what month and day?', 'supporting_facts': [['Park Seo-joon', 0], ['Park Seo-joon', 1], ['Fight for My Way', 0], ['Fight for My Way', 1], ['Fight for My Way', 2]], 'context': [['Robert Wightman', ['Robert Wightman is an American actor perhaps best known for replacing Richard Thomas in the role of John-Boy Walton in the TV series \"The Waltons\".', \" He played the role beginning with the show's eighth season in 1979 until the end of the series in 1981.\", ' He also appeared in the role in the TV movie \"A Day of Thanks on Walton\\'s Mountain\" in 1982.', ' His movie credits include \"American Gigolo\" and the starring role in \"Stepfather III\" as the main character of the film, taking over the role originally played by Terry O\\'Quinn.']], ['Fight for My Way', ['Fight for My Way (; lit.', ' \"Third-Rate My Way\") is a South Korean television series starring Park Seo-joon and Kim Ji-won, with Ahn Jae-hong and Song Ha-yoon.', ' It premiered on May 22, 2017 every Monday and Tuesday at 22:00 (KST) on KBS2.']], ['Khamani Griffin', ['Khamani Griffin (born August 1, 1998) is an American actor, who is best known for playing Bobby James in the UPN/CW series \"All Of Us\", and Tolee the Koala in \"Ni Hao, Kai-Lan\".', ' He starred as Ben Hinton in \"Daddy Day Care\" (2003) and had a role in \"Norbit\" (2007).', ' He has also appeared in \"Grey\\'s Anatomy,\" \"ER,\" and \"My Name Is Earl.\"', ' He has been nominated with three Young Artist Awards for his roles in \"Daddy Day Care\" and \"All of Us\".', \" He also made an appearance in Lil' Kim's video download.\", ' Khamani had a main role on the popular game show \"Are You Smarter Than A 5th Grader?', '\" until its series finale on September 18, 2009.']], ['Peggy Cartwright', ['Peggy Cartwright (November 14, 1912 – June 12, 2001) was a Canadian silent film actress perhaps best known for her short stint as the leading lady of the \"Our Gang\" comedies.', ' She appeared in four shorts in 1922 and, possibly, the pilot for the series, \"Our Gang\".', ' Cartwright only appeared in the first four Our Gang shorts that premiered in 1922.', ' \"One Terrible Day\" premiered on September 10, 1922, \"Fire Fighters\" premiered on October 8, 1922, \"Young Sherlocks\" premiered on November 26, 1922, and \"Saturday Morning\" premiered on December 3, 1922.']], ['The Fountain in the Park', ['\"The Fountain in the Park\", also known as \"While Strolling Through (or Thru\\') the Park One Day\", is a song by Ed Haley (1862–1932), published in 1884 by Willis Woodward & Co. of New York, but dating from about 1880.', ' It is best known for the being the source of the tune that contains the lyric \"While strolling through the park one day, in the merry merry month of May,\" and has been featured in numerous films, including \"Strike Up the Band\" (1940), in which it was sung by Judy Garland.']], ['Salmi Manja', ['Saleha binti Abdul Rashid (born 24 July 1937), better known by her pen name Salmi Manja, is a Malaysian novelist, poet, and journalist.', ' She was among the first Malaysian professional women writers and best known for her 1960 novel \"Hari Mana Bulan Mana\" (What Day What Month).', \" Femininity, women's issues, and Islam are recurring themes in her work.\"]], ['Hindsight (TV series)', ['Hindsight is an American comedy-drama television series that premiered on VH1 on January 7, 2015 and ended on March 11, 2015.', ' The series was created by Emily Fox and stars Laura Ramsey in the lead role of Becca Brady, who, while wrestling with doubts on the eve of her second wedding, finds herself sent back to 1995.', ' Specifically, Becca finds that she has time traveled to the day of her first wedding, a marriage that ultimately ended in divorce.', ' Upon her arrival in 1995, Becca reunites with her best friend Lolly (from whom she has become estranged in present day), breaks off her engagement to her first husband, and resolves to use her trip back in time to correct what she sees as personal and professional mistakes.']], ['Park Seo-joon', ['Park Seo-joon (born Park Yong-gyu) is a South Korean actor.', ' He is best known for his roles in the television dramas \"Kill Me, Heal Me\" (2015), \"She Was Pretty\" (2015), \"\" (2016-2017) and \"Fight for My Way\" (2017).']], ['Andrew Kaplan', ['Andrew Gary Kaplan is an American author, best known for his spy thriller novels.', ' He was born in Brooklyn, New York on May 18, 1941.', ' He went to Stuyvesant High School and Brooklyn College and after serving in the U.S. Army, he went to Europe and Africa, where he worked as a free-lance journalist and war correspondent for the \"International Herald Tribune\" in Paris.', ' He served in the Israeli Army during the Six Day War of 1967.', ' As a student leader in Israel, he helped start what was initially called \"the University of the Negev\" (today, Ben Gurion University of the Negev) and the Israeli Olympic fencing team.', ' After graduating in 1970 from Tel Aviv University, he earned his MBA at Oregon State University.', ' He has been a technology businessman and is the author of eight international best-selling novels, which have been translated into 21 languages around the world: \"Hour of the Assassins\", \"Scorpion\", \"Dragonfire\" (a main selection of the Book of the Month Club in Britain), \"War of the Raven\" and the other books of the Scorpion and Homeland series: \"Scorpion Betrayal\", \"Scorpion Winter\", \"Scorpion Deception\", and \"\", a bestselling original novel prequel to the hit award-winning \"Homeland\" television series .', ' His second book in the Homeland series, \"Homeland: Saul\\'s Game\", won the 2015 Scribe Award for Best Original Media Tie-in Novel.']], ['Michael Stephenson (filmmaker)', ['Michael Paul Stephenson (born February 28, 1978) is an American film director, producer, writer, and actor.', ' He is best known for his starring role in \"Troll 2\" (1990) and for directing the ensuing documentary, \"Best Worst Movie\" (2009).', ' His second documentary, \"The American Scream\" (2012), premiered on NBCUniversal\\'s Chiller network and was named a \"Must Watch\" by \"Entertainment Weekly\".', ' His latest film and narrative directorial debut, \"Girlfriend\\'s Day\" (2017), is a Netflix original film.', \" Premiering on Valentine's Day 2017, the comedy stars Bob Odenkirk, Amber Tamblyn, Natasha Lyonne and Stacy Keach.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.607\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab29486554299545a2cf9a4', 'answer': 'Alt-J', 'question': 'Which has more members, Dada or Alt-J?', 'supporting_facts': [['Dada (band)', 0], ['Dada (band)', 1], ['Alt-J', 0]], 'context': [['Dada (band)', ['Dada is a three piece rock band from California (United States).', ' The band is made up of Michael Gurley (guitar/co-lead vocals), Joie Calio (bass/co-lead vocals) and Phil Leavitt (drums).']], ['Juliette Roche', ['Juliette Roche (1884–1980) was a French painter and writer who associated with members of the Cubist and Dada movements.']], ['Kroesos Foundation', ['The Kroesos Foundation is an artistic collective set up by Swiss Artist, Mark Divo.', ' Between January and March 2002 they occupied the building in the centre of Zurich where the original European Dada movement began, as a response to the horrors of the first World War, which came to be known as the Cabaret Voltaire.', ' The collective organised a number of events/ performances over a period of three months until they were forced to leave the building.', ' In spite of their eviction they managed to have the building turned into a museum.', ' Members of the collective include Mark Divo, Aiana Calugar, Dan Jones, Lennie Lee, ingo giezendanner and Pastor Leumund Cult.', ' Throughout the winter of 2002 they were described as neo-Dadaists by the Swiss and international press.', ' The group have exhibited in a number of international exhibitions including the real Biennale at the Kinsky Palace in Prague']], ['Last Call Cleveland', ['Last Call Cleveland is a sketch comedy troupe.', ' The members originally met when they were students at Kent State University, where they produced a late-night cable show that aired on the student-run campus network TV2.', ' The show moved to Cleveland in 2001 to local broadcast cable station THE CAT (WAX 35 Cleveland and WAOH 29 Akron).', ' The group started doing live sketch comedy at the shortly lived Second City Cleveland, where they remain a sketch group today.', ' Although most of their performances have been in northeastern Ohio (primarily at Cabaret Dada, the Second City Cleveland theater and the House of Blues) they have also performed in Chicago at the Chicago Sketchfest, Washington, D.C. and in Los Angeles at the ImprovOlympic theater.']], ['The Holy Sisters of the Gaga Dada', ['The Holy Sisters of the Gaga Dada were an eclectic band originally from Santa Cruz, California, USA formed in 1981.', ' They were voted \"Best Alternative Rock Band of the Year\" by L.A. Weekly.', ' The collaboration of keyboardist Mary Jean Shaffer and guitarist Blancah Black, the Holy Sisters reveled in eccentric, quasi-religious imagery and feminist politics.', ' Other original members included Heidi Puckett (bass) and Jeff Grubic (tenor sax).', ' As the band morphed from conceptual entity to frequent club performers, Black left and two new members were added, Jill Fido (bass) and Charles Bingham (drums).', ' Kim Sockit later replaced Puckett, and Zero Jessephski, Jr. replaced Bingham, making the Holy Sisters of the Gaga Dada an all-female band.', ' The group would later move to Hollywood, California where they were featured in \"Once Upon Her Time,\" a TV program about women in the \\'80s which aired on the Lifetime Cablevision Network.']], ['7horse', ['7Horse is an American rock and blues duo formed in 2011 most notable for their song \"Meth Lab Zoso Sticker\" which was featured in Martin Scorsese\\'s film \"The Wolf of Wall Street\", in the second trailer and on the soundtrack.', ' It was also used by FoxSports on the pre-game show for the NFL playoff season.', ' The group consists of Phil Leavitt (songwriter, drummer, and lead vocals) and Joie Calio (songwriter, guitars, bass, and vocals).', ' Both are also members of the band dada.']], ['COUM Transmissions', ['COUM Transmissions were a music and performance art collective who operated in the United Kingdom from 1969 through to 1976.', ' Influenced by the Dada artistic movement, COUM were openly confrontational and subversive, challenging aspects of conventional British society.', ' Founded in Hull, Yorkshire by Genesis P-Orridge, other prominent early members included Cosey Fanni Tutti and Spydeee Gasmantell (also at school with Genesis P-Orridge).', ' Part-time member included Menzies, Haydn Robb, Les Maull (aka The Reverend Lelli), Ray Harvey and Fizzy Paet.', ' Later members included Peter \"Sleazy\" Christopherson and Chris Carter, who together with P-Orridge and Fanni Tutti went on to found the pioneering industrial band Throbbing Gristle in 1976.']], ['Shleu-Shleu', ['Shleu-Shleu is a kompa band formed the 22 December 1965 in Bas-Peut de Chose, Por-au-Prince by former members of the groupes Lorenceau and Memfoubins; and managed by Hugues \"Dada\" Jackaman (or Djakaman) a rich Arab Haitians businessman of Syrian origin and directed by on the ashes of \"Les Manfoubins\" created by Jean Baptiste, Jacques Vabre, Camille Philippe and Kiki Bayard and \"Following the demize of Les Frères Lorenceau\".', ' During their first appearance, Nemours Jean-Baptiste renamed them \"Mini Jazz\", due to their reduced format, thus unknowingly coining the term \"mini-jazz\", also referring to the mini-skirt fashion of the time.', ' The new band was composed of a solo saxophonist, Tony Moise, Jean-Claude Pierre-Charles (a.k.a. Peddy) and Hans Cherubin (a.k.a. Gro Bébé) on lead vocals.', ' After liven throughout Haiti, they achieved international notoriety, and in 1970, they were hired to travel to New York City to perform at Casa Borinquen.', ' They decided to stay in NYC.', ' They have played in many large American cities, spreading many of the musical gems of the Haitian diaspora.', ' In 1976, many of the original members of the band had to leave New York for several reasons.', ' This instability created a period of decline that lasted until 1991 when Jean-Baptiste Smith decided to revive the band with the addition of new musicians, such as the talented saxophonist Evens Latortue, guitarist Eddy Altine, percussionist Joseph Savius.']], ['Sonia Dada', ['Sonia Dada is an American rock/soul/rhythm and blues band, which tours with between six and eight members.', ' The Chicago-based band formed in 1990, when founding member Daniel Pritzker enlisted Michael Scott, Paris Delane, and Sam Hogan after hearing them sing in a subway station.', ' Sonia Dada has become a mainstay of the Chicago musical scene in the years since, incorporating elements of rock, soul, gospel, and funk.']], ['Alt-J', ['Alt-J, stylised as alt-J, are an English indie rock band formed in 2007 in Leeds, by Gwil Sainsbury (guitar/bass), Joe Newman (guitar/lead vocals), Thom Sonny Green (drums) and Gus Unger-Hamilton (keyboards/vocals).']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.608\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a881cbb55429938390d3ee7', 'answer': 'Hellenism', 'question': 'St. John of the Cross Episcopal Church has a rectory that was in a style that was a product of what earlier style?', 'supporting_facts': [['St. John of the Cross Episcopal Church', 3], ['Greek Revival architecture', 1]], 'context': [[\"St. John's Episcopal Church, Canandaigua\", ['The Episcopal presence in Canandaigua,New York begins about 1799 with the St. Matthew Society, a missionary group.', \" St. John's Episcopal Church was organized in 1814 and first met in the Ontario County, New York Court House in Canandaigua.\", \" St. John's erected a wooden church building in 1816.\", ' Bishop John Henry Hobart consecrated it that year.', ' The brick rectory was constructed alongside the church in 1851.', ' Steady congregational growth necessitated more space.', ' This led to the razing of the first church and the constructing of the current, larger stone Gothic building done in the parish church style popular in the 19th century.', ' Emlen T. Littel of New York, who was also the architect of Zion Episcopal Church (Palmyra, New York) designed the building.', ' This church was constructed in 1872 at a cost of $47,000 (approximately $850,000 today) and consecrated in 1886.', ' It contains several windows from the earlier wooden church, elaborate new stained glass windows imported from Europe, and one—The Parables Window—was designed by Daniel Cottier(1837–1891), who was considered an important influence on Louis Comfort Tiffany.', ' In 1908, new hardwood floors, choir stalls, and an organ were installed.', ' The parish house and a chapel were added at the same time.', ' In 1964-65, an addition to parish house included classrooms, a new chapel, and a dining/ meeting room added to celebrate the church’s 150th anniversary.', ' Recent additions include a columbarium with a capacity of 136 niches and a memorial garden, outside the church, reached through the columbarium and chapel in the south transept.']], ['Episcopal Diocese of the Virgin Islands', ['The Episcopal Diocese of the Virgin Islands is a diocese of the Episcopal Church in the United States of America (ECUSA) which includes both the United States Virgin Islands and the British Virgin Islands.', ' The diocese is a part of Province II of the Episcopal Church.', ' The current Diocesan Bishop of the Virgin Islands is the Edward Ambrose Gumbs.', ' The cathedral church of the diocese is the Cathedral Church of All Saints, Charlotte Amalie.', ' The diocese currently comprises 14 churches.', \" There is a functioning parish school on St. Thomas All Saints Cathedral School there was an academic campus on St. Croix, St. Dunstan's Episcopal High School.\", \" St. Dunstan's closed in the 1990s.\", \" There is also the St. Georges School located on the parish property of St. Georges Episcopal Church in Road Town, Tortola in the British Virgin Islands, which also opened the St. Georges School (Secondary Division) in Palestina Estate near to the St. Paul's Episcopal Church in Sea Cow's Bay, Tortola in the British Virgin Islands.\", \" There is also the St. Mary's School located on the parish property of the St. Mary's Episcopal Church in Valley, Virgin Gorda in the British Virgin Islands.\"]], [\"St. Luke's Church and Cemetery\", [\"St. Luke's Episcopal Church and Cemetery is a historic Episcopal church complex, cemetery, and national historic district located at 303-321 N. Cedar Street, 322 E. McBee Street in Lincolnton, Lincoln County, North Carolina.\", ' The complex includes the church, parish hall, and rectory.', ' The church was built in 1885-1886, and is a Late Gothic Revival style frame structure with a brick veneer added in 1922-1923.', ' The tower is believed to date to 1859.', ' The parish hall was built in 1907, and is a one-story, rectangular frame building.', ' The rectory was built in 1911-1912, and is a two-story, \"T\"-form Colonial Revival style dwelling with a pebbledash finish.', ' The cemetery includes approximately 300 gravestones, with the earliest dating to 1854.']], ['Mountain Road Historic District', ['Mountain Road Historic District is a national historic district in Halifax, Halifax County, Virginia.', ' The district includes 22 contributing buildings located along Mountain Road (State Route 360) and consists of two churches, a parish hall, a masonic hall, and a host of private residences dating to the 19th and early 20th centuries.', \" Notable buildings include the Masonic Lodge (1828), Methodist Church (1831), St. John's Episcopal Church (1844), Magnolia Hill, Grand Oaks, and St. John's Rectory.\", \" Several of the earlier dwellings and St. John's Episcopal Church were designed by Dabney Cosby, Jr., son of the Jeffersonian workman, Dabney Cosby, Sr.\"]], ['St. John of the Cross Episcopal Church', ['St. John of the Cross Episcopal Church, Rectory and Cemetery is a historic Episcopal church complex located at Bristol, Elkhart County, Indiana.', ' The church was built between 1843 and 1847, and is a one-story, Gothic Revival style frame building.', ' It has a projecting bell tower with octagonal roof and lancet windows.', ' The associated rectory was built in 1830, and is a 1 1/2-story, rectangular, Greek Revival style frame dwelling.', ' The complex also includes the contributing church cemetery.']], [\"St. Augustine's Episcopal Church Complex\", [\"St. Augustine's Episcopal Church Complex is a historic Episcopal church complex at 6 Old Post Road north of Croton-on-Hudson, Westchester County, New York.\", ' The complex consists of the church and rectory The church consists of the original building and a later parish hall connected by an enclosed hyphen.', ' The church was built in 1857, the parish hall was added in 1882, and the rectory was completed in 1910.', ' The church and parish hall are in the Gothic Revival style, while the rectory is in the Colonial Revival style.']], ['St. Barnabas Episcopal Church (Troy, New York)', ['St. Barnabas Episcopal Church, later called Christ & St. Barnabas Episcopal Church, and now known as New Hope Missionary Baptist Church, is an historic Episcopal church and rectory at 2900 Fifth Avenue in Troy, Rensselaer County, New York.', ' The church was built in 1895 and is a red brick church in the Late Gothic Revival style.', ' It has a gable roof and three hipped dormers.', ' It has an open bell tower and slender conical turrets.', ' It features a rose window depicting the Madonna and Child.', ' The former rectory is a 2\\xa0⁄ -story, L-shaped brick residence.', ' Also on the property is a contributing carved stone crucifixion dated to about 1900.']], ['Greek Revival architecture', ['The Greek Revival was an architectural movement of the late 18th and early 19th centuries, predominantly in Northern Europe and the United States.', ' A product of Hellenism, it may be looked upon as the last phase in the development of Neoclassical architecture.', ' The term was first used by Charles Robert Cockerell in a lecture he gave as Professor of Architecture to the Royal Academy of Arts, London in 1842.']], [\"St. Peter's Episcopal Church and Rectory\", [\"St. Peter's Episcopal Church and Rectory is a historic Episcopal church and rectory at 36-38 W. Campbell Street in Blairsville, Indiana County, Pennsylvania.\", ' The church was built in 1830, and is a small, rectangular brick building on a stone foundation in an Early Gothic Revival style.', ' It features a belfry atop the front entrance gable roof.', ' The rectory was built in 1889, and is a 2\\xa01/2-story, wood frame building with Eastlake Movement elements.']], ['Old Rectory (Perrowville, Virginia)', [\"Old Rectory of St. Stephen's Episcopal Church is a historic Episcopal church rectory located near Perrowville, Bedford County, Virginia.\", ' It was built in 1787, and is a \"T\"-shaped frame dwelling with exterior end chimneys and a gable roof.', ' It features a modern one bay, two-story portico supported by four fluted Doric order columns.', \" From around 1828 to 1904, the house served as the rectory of St. Stephen's Episcopal Church.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.609\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5add3b035542997545bbbcce', 'answer': 'November 2017', 'question': 'The Gate is a song from the album scheduled for release in what month?', 'supporting_facts': [['The Gate (song)', 1], ['Utopia (Björk album)', 0]], 'context': [['Killer BiSH', ['Killer BiSH is the third album by Japanese idol group BiSH released through Avex Trax on October 5, 2016.', ' The album is the first full album by the group to be released from a major label.', ' The album is the first release to feature new member Ayuni D, who joined the group in August 2016 following the departure of Hug Mii.', ' Unlike their independent label albums, songs from the album were not released as free downloads prior to release.', ' However, the album was released in its entirety in digital format a month prior to the physical release.', \" The first 24 hours of the digital release had all of the group's releases including KiLLER BiSH priced at 300 yen.\", ' The album was preceded by the single \"DEADMAN\", released on May 4, 2016.', ' The music video for the track \"Orchestra\" was posted on Youtube on September 9.', ' RUKA from Visual-kei band Nightmare, a well known fan of BiSH and BiS, provided the music for the track \"IDOL is SHiT\".', ' The track title is an homage to BiS\\'s album IDOL is DEAD while the track is an homage to the song \"IDOL\" from that album.']], ['American Spring (Anti-Flag album)', [\"American Spring is Anti-Flag's ninth studio album.\", ' The album was released on May 26, 2015.', \" The album marks the band's first release for Spinefarm Records after releasing their previous two albums on SideOneDummy Records.\", ' A lyric music video for the album\\'s first single, \"Fabled World\" was released in March 2015.', ' The second single \"Sky Is Falling\" was released a month later.', ' On May 19, A music video for Brandenburg Gate was released.']], ['Music for Cars (EP)', ['Music for Cars is the third extended play by English rock band The 1975, released on 4 March 2013 through Dirty Hit.', ' and in the United States through Vagrant 5 March.', \" It is the third of four EPs released before the band's self-titled debut.\", \" It is also confirmed to share its title with the band's planned third album scheduled for release in 2018.\"]], ['Utopia (Björk album)', ['Utopia is the upcoming ninth studio album by Icelandic musician Björk, scheduled for release in November 2017 through One Little Indian Records.', ' During a Facebook Live event held on 15 September 2017, the release date of the album\\'s first single, Björk gave insight into the album\\'s theme of utopia in an uncertain political environment: \"We have Trump, we have Brexit, we have our issues in Iceland, we have our environmental issues.', ' I think if there ever was an urgency or necessity to come up with another Utopian model, how we\\'re going to live our lives, I think it\\'s now, and (these are) my proposals.\"']], [\"Joker's Daughter (band)\", ['Joker\\'s Daughter was originally a musical collaboration between Greek-English singer/songwriter Helena Costas and producer Danger Mouse who released their debut album \"The Last Laugh\" in 2009.', ' Today Helena Costas is backed by a new band and has a new album scheduled for release on 31 October 2011.']], ['Change (Andrew Hill album)', ['Change is the ninth album by American jazz pianist Andrew Hill featuring performances recorded and scheduled for release in 1966 on the Blue Note label.', ' The album was originally scheduled for issue in 1967 as BST 84233, but was held back from release until 1975, when the tracks appeared under Sam Rivers\\' name, as part of the double LP set \"Involution\", which combined them with tracks recorded under Rivers\\' leadership which would eventually see release as \"Dimensions & Extensions\".', ' The first release under Hill\\'s name occurred in 1995 as part of the Mosaic box set \"The Complete Blue Note Andrew Hill Sessions (1963-66)\".', \" The album features Hill's quartet performing six original compositions.\", ' In 2007, two alternate takes, previously included in the Mosaic set, were added to the 2007 CD release.', ' The first one, a shorter take of \"Violence\", was initially chosen as master track for the piece.']], ['Pacific Myth', ['Pacific Myth is a subscription-based serial album and the fifth major release by the Canadian progressive metal band Protest the Hero, distributed through the online music subscription service Bandcamp.', ' Following the independent release of their crowdfunded 2013 album \"Volition\", the band sought to explore alternative release methods and decided upon a subscription-like platform wherein subscribers could get access to the scheduled arrival of a new song every month.', ' Each of the songs, including artwork, lyrics, liner notes, instrumental versions, and high-quality downloads, were released through Bandcamp each month starting on October 15, 2015 with the first track, \"Ragged Tooth,\" and ending on March 15, 2016 with the final track, \"Caravan.\"', \" The six songs effectively make up the band's fifth EP, and 11th overall release.\", \" It is the band's only release with Cam McLellan on bass and the first with Mike Ieradi on drums.\"]], ['Rex Riot', ['Rex Riot, born Nicholas Rex Valente, is an electronic music producer.', ' He is known for his work with Nintendo, for their 2012 Wii U Campaign, and various work including a popular remix of Kanye West\\'s \"All of the Lights\", which he produced with Infuze.', ' He currently has releases under Play Me Records, Heavy Artillery Records, and an upcoming album scheduled for release in summer 2013.']], ['Chapter 1 (EP)', ['Chapter 1 is the second extended play (EP) by American country music singer Kane Brown who is signed with Sony Music Nashville in early 2016.', ' The five-song EP was released on March 18, 2016, as his first EP with the Sony label although he had an earlier independently released EP on his own label titled \"Closer\".', ' \"Chapter 1\" is considered a prelude to his debut studio album scheduled later in 2016.']], ['The Gate (song)', ['\"The Gate\" is a song recorded by Icelandic musician Björk.', ' It was released on 15 September 2017 through One Little Indian as the lead single from her ninth studio album, \"Utopia\" (2017).', ' The song was written and produced by Björk and Arca.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.609\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab3c53955429969a97a81b9', 'answer': 'David Jolly', 'question': 'Who won the special election in 2014 to replace Bill Young?', 'supporting_facts': [['Rod Smith (politician)', 3], ['Alex Sink', 3]], 'context': [['David Jolly', [\"David Wilson Jolly (born October 31, 1972) is an American attorney, former lobbyist, and former U.S. Representative for Florida's 13th congressional district.\", ' A Republican, he previously served as general counsel to his predecessor, U.S. Rep. Bill Young.', \" He won the race for Young's seat in a 2014 special election against Democrat Alex Sink.\", ' He was subsequently re-elected in November 2014, winning 75 percent of the vote, but was unseated in 2016 by former Governor Charlie Crist.', ' Jolly has become a prominent Republican critic of U.S. President Donald Trump.']], ['Alex Sink', ['Adelaide Alexander \"Alex\" Sink (born June 5, 1948) is an American politician and financier.', ' A member of the Democratic Party, Sink was the Chief Financial Officer for the state of Florida and treasurer on the board of trustees of the Florida State Board of Administration.', ' She was the Democratic nominee for Governor of Florida and faced Republican nominee Rick Scott in the 2010 Florida gubernatorial election, losing to Scott by a 1% margin.', ' Sink was the unsuccessful Democratic candidate in the special election, losing to Republican David Jolly on March 11, 2014, in a race to fill the vacancy created by the death in 2013 of U.S. Representative Bill Young.']], ['Ed Neilson', ['Edward \"Ed\" Neilson is an American politician and member of the Democratic Party.', ' In April 2012, he won a special election to represent the 169th District in the Pennsylvania House of Representatives.', ' In May 2014 he won a special election to serve as an at-large member of Philadelphia City Council replacing outgoing councilmember Bill Green.', ' In August 2015, he won a special election to represent the 174th District in the Pennsylvania House of Representatives.']], ['Born Again Tour 1983', ['The Born Again Tour 1983 was a global concert tour by in support of Black Sabbath\\'s \"Born Again\" album.', \" Both the album and the tour were the only ones of Black Sabbath's to feature former Deep Purple frontman Ian Gillan on lead vocals.\", ' Ex-Electric Light Orchestra drummer Bev Bevan was hired to replace Bill Ward, who had returned to the band for the recording of the album after a two-year hiatus, for the tour.', ' This was the final tour to feature original Black Sabbath bassist Geezer Butler until 1992\\'s \"Dehumanizer\" tour.']], ['Louisiana State Treasurer special election, 2017', ['The Louisiana State Treasurer special election will take place on October 14, 2017, to elect the State Treasurer of Louisiana, with a runoff election to be held on November 18, 2017, if necessary.', ' Incumbent Republican State Treasurer John Kennedy was elected to the U.S. Senate in 2016.', ' First Assistant Treasurer Ron Henson replaced Kennedy as Treasurer, and will serve until the special election.', ' Henson will not run in the special election.']], ['C. W. Bill Young Regional Reservoir', ['The C.W. Bill Young Regional Reservoir is a 15.5 e9USgal reservoir which collects water from the Alafia and Hillsborough Rivers in central Florida.', \" It is named for C.W. Bill Young, the U.S. Congressman from Florida's 10th congressional district.\", ' Tampa Bay Water, the regional water authority for Hillsborough, Pinellas and Pasco counties, worked for nearly a decade in constructing the reservoir, which was completed in June 2005, and officially opened on 15 October 2005.']], ['Terry Tornek', ['Terry Eliot Tornek (born November 23, 1945) is an American politician, and the mayor of Pasadena, California.', ' He previously served on the Pasadena City Council.', \" On April 21, 2015 he defeated City Councilmember Jacque Robinson in the general election to replace Bill Bogaard, the longest serving mayor in Pasadena's history.\"]], ['Electoral history of Ed Markey', ['This is the electoral history of Ed Markey, a Democratic Senator from Massachusetts.', ' He was previously a Democratic Representative from Massachusetts, representing the 7th and 5th districts.', ' Markey was first elected in a 1976 special election to replace the deceased Torbert Macdonald, and was re-elected in every subsequent election.', ' He was also the Democratic candidate, and winner, of the 2013 special election, for the United States Senate.']], ['Rod Smith (politician)', ['Rodney Warren Smith (born November 15, 1949) is an American politician from the U.S. state of Florida.', ' A Democrat, Smith was a member of the Florida Senate from Gainesville from 2001 until 2006.', ' Smith ran for the Democratic nomination for Governor of Florida in the 2006 election but lost to Congressman Jim Davis.', ' In 2010, Smith was the Democratic nominee for Lieutenant Governor of Florida as the running mate of Alex Sink in her campaign for Governor of Florida.', ' From November 2010 through January 2013 Smith served as chairman of the Florida Democratic Party.', ' In 2016, Smith again ran for a seat in the Florida Senate but was ultimately defeated by Keith Perry, a Republican and former State Representative.']], ['San Diego mayoral special election, 2005', ['The 2005 San Diego mayoral special election was a special election held on Tuesday, November 8, 2005, to elect the mayor for San Diego.', ' The special election was necessary due to the resignation of former Mayor Dick Murphy.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.611\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a89ec7b5542992e4fca8483', 'answer': 'Hungarian Rhapsody No. 2', 'question': 'A Car-Tune Portrait was released on June 26, 1937, the cartoon gives an imaginative take on which second in a set of works by composer Franz Liszt?', 'supporting_facts': [['A Car-Tune Portrait', 1], ['Hungarian Rhapsody No. 2', 0]], 'context': [['Lina Ramann', ['Lina Ramann (July 24, 1833 – March 30, 1912) was a German writer and teacher known for her books on the Hungarian composer and pianist Franz Liszt.', ' During 1874-94, she wrote his \"official\" though inaccurate three volume biography \"\"Franz Liszt, the artist and man\"\" (Franz Liszt als Künstler und Mensch) which was published between 1880-94.']], ['Cosima Wagner', [\"Cosima Wagner (born Francesca Gaetana Cosima Liszt; 24 December 1837\\xa0– 1 April 1930) was the illegitimate daughter of the Hungarian pianist and composer Franz Liszt and Marie d'Agoult.\", ' She became the second wife of the German composer Richard Wagner, and with him founded the Bayreuth Festival as a showcase for his stage works; after his death she devoted the rest of her life to the promotion of his music and philosophy.', ' Commentators have recognised Cosima as the principal inspiration for Wagner\\'s later works, particularly \"Parsifal\".']], ['A Car-Tune Portrait', ['A Car-Tune Portrait is a cartoon in the Color Classics series produced by Fleischer Studios.', ' Released on June 26, 1937, the cartoon gives an imaginative take on Franz Liszt\\'s \"Hungarian Rhapsody No. 2\".']], ['József Ács (musician)', ['József Ács (born 1948) is a German composer, and classical pianist and composer of Hungarian origin.', ' A graduate of the Franz Liszt Academy of Music in Budapest and the Robert Schumann Hochschule in Düsseldorf, he won the first prize for piano at the \"German Music Competition\" in the Beethovenhalle in Bonn.', ' He is particularly renowned for his recitals of Franz Liszt, and has done work in conjunction with the Vatican Archives.', ' More recently he has been performing the works of Italian composer Ruggero Leoncavallo.', \" He composed a completion of Leoncavallo's Requiem which was a fragment.\", ' Ács also wrote a small mass for choir and organ called, \"Weihnachtslieder-Messe.\"', ' Appropriate for the Christmas season, this joyful mass incorporates two Christmas tunes, In dulci jubilo, and Es kommt ein Schiff geladen.']], ['Hungarian Rhapsody No. 2', ['Hungarian Rhapsody No. 2 in C-sharp minor, S.244/2, is the second in a set of 19 Hungarian Rhapsodies by composer Franz Liszt, and is by far the most famous of the set.']], ['Liszt (crater)', ['Liszt is a crater on Mercury.', ' It has a diameter of 85 kilometers.', ' Its name was adopted by the International Astronomical Union in 1985.', ' Liszt is named for the Hungarian composer Franz Liszt, who lived from 1811 to 1886.']], ['Hans von Bülow', ['Baron Hans Guido von Bülow (January 8, 1830February 12, 1894) was a German conductor, virtuoso pianist, and composer of the Romantic era.', ' One of the most famous conductors of the 19th century, his activity was critical for establishing the successes of several major composers of the time, especially Richard Wagner and Johannes Brahms.', \" Alongside Carl Tausig, Bülow was perhaps the most prominent of the early students of Hungarian virtuoso pianist, conductor and composer Franz Liszt – therein performed the first public performance of Liszt's Sonata in B minor in 1857.\", \" He became acquainted with, fell in love and eventually married Liszt's daughter Cosima, who later left him for Wagner.\", ' Noted for his interpretation of the works of Ludwig van Beethoven, he was one of the earliest European musicians to tour the United States.']], ['Budapest Ferenc Liszt International Airport', ['Budapest Ferenc Liszt International Airport (Hungarian: \"Budapest Liszt Ferenc Nemzetközi Repülőtér\" ) (IATA: BUD,\\xa0ICAO: LHBP) , formerly known as \"Budapest Ferihegy International Airport\" and still commonly called just Ferihegy, is the international airport serving the Hungarian capital city of Budapest, and by far the largest of the country\\'s four commercial airports.', ' The airport is located 16 km southeast of the center of Budapest (bordering Pest county) and was renamed in 2011 in honor of the most famous Hungarian composer Franz Liszt (Hungarian Liszt Ferenc) on the occasion of the 200th anniversary of his birth.', ' The airport won the Skytrax Best Eastern European airport prize three times in a row (2014-2016).']], ['Eduard Reuss', ['Eduard Reuss (16 September 1851 - 18 February 1911) was a German composer, pianist, music educator, and writer on music.', ' He is best known for his writings on composer Franz Liszt, including a highly thought of biography \"Ein Lebensbild\" (1898).', ' He also authored the book \"Liszts Lieder\" (1906) and penned several essays on the life and works of Liszt.', ' As a composer, he mainly produced works for solo piano and also made arrangements of several works by Liszt.']], ['O lieb, so lang du lieben kannst', ['O lieb, so lang du lieben kannst is a poem written by Ferdinand Freiligrath, a 19th-century German writer.', ' In 1847, Hungarian composer Franz Liszt set the poem to music (soprano voice and piano), and eventually adapted it into his famous Liebesträume No. 3.', \" The work is one of Liszt's most famous and poignant.\", ' \"Liebesträume\" in German means \"Dreams of Love\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.611\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7d6a2d5542991319bc93c6', 'answer': 'Southern Isles', 'question': 'King of Norway Magnus Barefoot launched aggressive military campaigns in this region that was at times independent of external control and was known to the Norse as what?', 'supporting_facts': [['Magnus Barefoot', 0], ['Magnus Barefoot', 1], ['Kingdom of the Isles', 1], ['Kingdom of the Isles', 4]], 'context': [['Magnus Barefoot', ['Magnus Olafsson (Old Norse: \"Magnús Óláfsson\", Norwegian: \"Magnus Olavsson\"; 1073 – 24 August 1103), better known as Magnus Barefoot (Old Norse: \"Magnús berfœttr\", Norwegian: \"Magnus Berrføtt\"), was King of Norway (as Magnus III) from 1093 until his death in 1103.', ' His reign was marked by aggressive military campaigns and conquest, particularly in the Norse-dominated parts of the British Isles and Ireland, where he extended his rule to the Kingdom of the Isles and Dublin.']], ['Olaf Magnusson of Norway', ['Olaf Magnusson (1099 – 22 December 1115) was king of Norway 1103–1115.', ' He was the son of King Magnus Barefoot and Sigrid, daughter of Saxe of Vik.']], ['Battle of Stamford Bridge', [\"The Battle of Stamford Bridge took place at the village of Stamford Bridge, East Riding of Yorkshire, in England on 25 September 1066, between an English army under King Harold Godwinson and an invading Norwegian force led by King Harald Hardrada and the English king's brother Tostig Godwinson.\", ' After a bloody battle, both Hardrada and Tostig along with most of the Norwegians were killed.', ' Although Harold Godwinson repelled the Norwegian invaders, his army was defeated by the Normans at Hastings less than three weeks later.', ' The battle has traditionally been presented as symbolising the end of the Viking Age, although major Scandinavian campaigns in Britain and Ireland occurred in the following decades, such as those of King Sweyn Estrithson of Denmark in 1069–1070 and King Magnus Barefoot of Norway in 1098 and 1102–1103.']], [\"King Magnus' Halt railway station\", [\"King Magnus' Halt, sometimes referred to as Magnus' Grave, is the terminal railway station on the Downpatrick & County Down Railway's Southern Line, located in the town of Downpatrick in County Down, Northern Ireland.\", ' It takes its name from the nearby grave of Viking King Magnus Barefoot.', ' The grave is a Downpatrick tourist attraction that was not easily accessible prior to the arrival of the railway.', ' It features a platform with lampposts on the East side of the line, which looks onto a field containing the barrow of King Magnus and a runestone which was placed in March 2003 to mark the 900th anniversary of his death.']], ['Radama I', ['Radama I \"the Great\" (1793–1828) was the first Malagasy sovereign to be recognized as King of Madagascar (1810-1828) by a European state.', ' He came to power at the age of 18 following the death of his father, King Andrianampoinimerina.', \" Under Radama's rule and at his invitation, the first Europeans entered his central highland Kingdom of Imerina and its capital at Antananarivo.\", ' Radama encouraged these London Missionary Society envoys to establish schools to teach tradecraft and literacy to nobles and potential military and civil service recruits; they also introduced Christianity and taught literacy using the translated Bible.', ' A wide range of political and social reforms were enacted under his rule, including an end to the international slave trade, which had historically been a key source of wealth and armaments for the Merina monarchy.', ' Through aggressive military campaigns he successfully united two-thirds of the island under his rule.', ' Abuse of alcohol weakened his health and he died prematurely at age 35.', ' He was succeeded by his highest-ranking wife, Ranavalona I.']], ['Battle of Anglesey Sound', ['The Battle of Anglesey Sound was fought in June or July 1098 on the Menai Strait (\"Anglesey Sound\"), separating the island of Anglesey from mainland Wales.', \" The battle was fought between Magnus Barefoot, King of Norway, and the Anglo-Norman earls Hugh of Montgomery and Hugh d'Avranches, and took place as part of Magnus Barefoot's expedition into the Irish Sea, which sought to assert Norwegian rule over the Kingdom of the Isles.\"]], ['Magnus Barefoot Cinema Centre', ['Magnus Barefoot Cinema Centre (Norwegian: \"Magnus Barfot kinosenter\") is a cinema multiplex with 5 screens owned by Bergen Cinema.', ' It is named after the street that runs along the building (however, it is not the address of the complex), which in turned is named after the Norwegian king Magnus Barefoot.', ' It is the main venue of Bergen International Film Festival.']], ['List of rulers of the Kingdom of the Isles', ['The Kingdom of the Isles comprised the Hebrides, the islands of the Firth of Clyde and the Isle of Man from the 9th to the 13th centuries AD.', ' The islands were known to the Norse as the \"Suðreyjar\", or \"Southern Isles\" as distinct from the \"Norðreyjar\" or Northern Isles of Orkney and Shetland.', ' The historical record is incomplete and the kingdom was probably not a continuous entity throughout the entire period.', ' The islands concerned are sometimes referred to as the \"Kingdom of Mann and the Isles\", although only some of the later rulers claimed that title.', ' At times the rulers were independent of external control, although for much of the period they had overlords in Norway, Ireland, England, Scotland or Orkney.', ' At times there also appear to have been competing claims for all or parts of the territory.', ' The islands involved have a total land area of over 8300 km2 and extend for more than 500 km from north to south.']], ['Kingdom of the Isles', ['The Kingdom of the Isles comprised the Hebrides, the islands of the Firth of Clyde and the Isle of Man from the 9th to the 13th centuries AD.', ' The islands were known to the Norse as the \"Suðreyjar\", or \"Southern Isles\" as distinct from the \"Norðreyjar\" or Northern Isles of Orkney and Shetland.', ' The historical record is incomplete, and the kingdom was not a continuous entity throughout the entire period.', ' The islands concerned are sometimes referred to as the Kingdom of Mann and the Isles, although only some of the later rulers claimed that title.', ' At times the rulers were independent of external control, although for much of the period they had overlords in Norway, Ireland, England, Scotland or Orkney.', ' At times there also appear to have been competing claims for all or parts of the territory.', ' The islands involved have a total land area of over 8300 km2 and extend for more than 500 km from north to south.']], ['Norman invasion of Wales', ['The Norman invasion of Wales began shortly after the Norman conquest of England under William the Conqueror, who believed England to be his birthright.', ' Initially (1067–1081), the invasion of Wales was not undertaken with the fervor and purpose of the invasion of England.', \" However, a much stronger Norman invasion began in 1081 and by 1094 most of Wales was under the control of William's eldest son, King William II of England.\", ' The Welsh greatly disliked the \"gratuitously cruel\" Normans and by 1101 had regained control of the greater part of their country under the long reign of King Gruffudd ap Cynan, who had been imprisoned by the Normans for twelve years before his escape.', ' Gruffudd had some indirect help from King Magnus III of Norway (Magnus Barefoot) who attacked the Normans briefly off the Isle of Anglesey in northwest Wales near Ynys Seiriol, killing Hugh of Montgomery, 2nd Earl of Shrewsbury and leaving the Normans depleted and demoralized.', ' Magnus went on to take the Orkney Islands, the Hebrides, and the Isle of Man, islands north of Wales and west and north of Scotland and England, in 1098.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.611\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab5d43f5542997d4ad1f1f6', 'answer': 'Polish independence', 'question': \"The musical composer of Manru was a spokesman for what country's independence?\", 'supporting_facts': [['Manru', 0], ['Ignacy Jan Paderewski', 0]], 'context': [['Manru', ['Manru is an opera (lyrical drama) in three acts, music by Ignacy Jan Paderewski composed to the libretto by Alfred Nossig (English translation by Henry Edward Krehbiel), based on the novel \"A Hut Behind the Village\" (1843) by Józef Ignacy Kraszewski.']], ['David Nessim Lawrence', ['David N. Lawrence (born 1960) is an American musical composer known primarily for his work on television and movie scores.', ' In 2002, he won an ASCAP award for his work on \"American Pie 2\".', ' He also composed music for the Disney Channel Original Movies \"High School Musical\", \"The Cheetah Girls 2\", \"High School Musical 2\", \"High School Musical 3\", \"Jericho (TV Series by CBS 2006)\" and \"\".', ' He also composed additional music for three Sesame Street videos like \"Elmo Says BOO!', ',\" \"Big Bird Gets Lost,\" and \"Let\\'s Eat!', ' Funny Food Songs,\" where he was credited as David Lawrence.']], ['Edward E. Rice', ['Edward Everett Rice (December 21, 1847 – November 16, 1924) was an American musical composer and theater producer active during the late 19th and early 20th Centuries, known primarily for being a pioneer of American musical theater and introducing to Broadway a musical by African-American writers and performers.']], ['Alexander Laszlo (composer)', ['Alexander Laszlo (November 22, 1895 Budapest (Hungary) - November 17, 1970 Los Angeles, California) was a Hungarian-American pianist, musical composer, arranger and inventor.', ' He was born Sandor (\"San\") Totis, but used the professional name of Alexander Laszlo as a composer and music publisher.']], ['Composer laureate', ['A composer laureate is a position awarded by a government as an honor to a musical composer.']], ['Scott Glasgow', ['Scott Glasgow is a Hollywood-based musical composer.', ' Has a Bachelor of Music from California State University, Northridge and a Master of Music from the San Francisco Conservatory of Music in 2001 where he was a student of Conrad Susa.', ' Scott studied with John Corigliano at the Aspen Music Festival 2002 and was in the ASACP Film Scoring program in 2004.', ' Since 2005 has made his career as a film composer with over 20+ studio features to date.', ' Scott also teaches at the university level, with classes in film scoring at CSUN California State University, Northridge and UCLA University of California, Los Angeles.', ' Scott has contributed addition music to films, TV and games including video games DC Universe Online, TV shows HBO Curb Your Enthusiasm and CBS 60 Minutes and feature films .']], ['Marianne de Pury', ['Marianne de Pury (born 3 April 1935) is a theatre artist and composer born in St. Gallen, Switzerland.', ' She is best known as the musical composer of two 1966 anti-war plays, Jean-Claude van Itallie\\'s social satire \"America Hurrah\" and Megan Terry\\'s rock musical \"Viet Rock\".']], ['Ignacy Jan Paderewski', ['Ignacy Jan Paderewski, GBE (] ; 18 November [O.S. 6 November] 1860  – 29 June 1941) was a Polish pianist and composer, politician and spokesman for Polish independence.', ' He was a favorite of concert audiences around the world.', ' His musical fame opened access to diplomacy and the media.']], ['Márton Vizy', ['Marton Vizy (born December 2, 1977) is a Hungarian pop singer-songwriter, musical composer from Budapest, Hungary.', ' Marton is the composer of the Hungarian hit musical Én, József Attila (Me, Attila József), which performed at the Madach Theatre, Budapest.']], ['Philippine Idol', ['Philippine Idol is the first version of the Idol series in the Philippines, the 35th country in the world to air a local \"Idol\" adaptation and the sixth in Asia.', ' The franchise was first awarded by FremantleMedia, 19 Entertainment and CKX, Inc. to ABC Development Corporation (ABC now TV5) in 2006 and then to GMA Network in 2008.', ' Just like the premise of original show \"Pop Idol\", \"Philippine Idol\" aims to find the best singer in the country who can be defined as the \"national\" singer.', ' Local television personality Ryan Agoncillo hosted the program.', \" Ryan Cayabyab (musical composer), Pilita Corrales (singer, known as Asia's Queen of Songs) and Francis Magalona (rapper and producer) were also judges of the show.\", \" Agoncillo, Corrales and Magalona auditioned to be part of the program, while Cayabyab was chosen by the program's producers.\", ' Meanwhile, actress Heart Evangelista hosted the daily updates program \"I ♥ Philippine Idol: Exclusive\".', \" Composer Mel Villena was the show's musical director.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.612\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7c221f554299683c1c62d2', 'answer': 'Marlborough', 'question': 'New Hampshire Route 124 runs between two regions, one of which is found in Cheshire County, New Hampshire, United States with the population of about 2000 and is named what?', 'supporting_facts': [['New Hampshire Route 124', 1], ['Marlborough, New Hampshire', 0], ['Marlborough, New Hampshire', 1]], 'context': [['Marlborough, New Hampshire', ['Marlborough is a town in Cheshire County, New Hampshire, United States.', ' The population was 2,063 at the 2010 census.', ' The town is home to the Kensan-Devan Wildlife Sanctuary at Meetinghouse Pond.']], ['Winchester Town Hall (New Hampshire)', ['The town hall of Winchester, New Hampshire, is located on Main Street, just south of the junction of New Hampshire Route 10 and New Hampshire Route 119 in the center of the town.', ' Built of brick in 1911-12 to a design by S. Winthrop St. Clair, it is the only Gothic Revival town hall in Cheshire County.', ' It was built because the previous town meeting house (a combination town hall and church at the site of the Winchester Memorial Church) was destroyed by fire.', ' The church was also designed by St. Clair, a Boston-based architect who hailed from Winchester.']], ['Jaffrey Center Historic District', ['The Jaffrey Center Historic District encompasses the traditional civic heart of the small town of Jaffrey, New Hampshire.', \" The district lies to the west of the Jaffrey's main business district, extending along Main Street (New Hampshire Route 124) from Harkness Road to the Jaffrey Common, and along Thorndike Pond Road northward from Main Street.\", \" The district retains the feel of an 18th-19th century rural village, and includes elements dating to shortly after the town's incorporation in 1773.\", ' The old burying ground was established in 1774, and the old meeting house (now a cultural center) was raised in 1775.', ' The district includes 19th century school houses, and houses that were built in the 18th and 19th centuries, in predominantly Georgian, Federal, and Greek Revival styles.', \" One of the town's early industries is also represented, in the remnants of a tannery established c. 1810 at the east end of the district.\"]], ['New Jersey Route 124', ['Route 124 is a state highway in the northern part of New Jersey in the United States that is 14.74 mi long.', ' It is the eastern section of what used to be Route 24 before that road was realigned to its current freeway alignment.', ' The western end is at an intersection with U.S. Route 202 (US\\xa0202) and County Route 510 (CR\\xa0510) in Morristown, Morris County; the eastern end continues as CR\\xa0603 on Springfield Avenue at the border between Maplewood and Irvington in Essex County.', ' The route runs through suburban areas of Morris County, passing through Madison and Chatham.', ' It interchanges with Route 24 on the border of Millburn, Essex County and Summit, Union County and serves as a frontage road for that route.', ' Upon splitting from Route 24, Route 124 continues east through Springfield Township, Union Township, and Maplewood to its eastern terminus.']], ['New Hampshire Route 32', ['New Hampshire Route 32 (NH 32) is a state highway in the U.S. state of New Hampshire.', ' The highway runs 14.139 mi from the Massachusetts state line in Richmond, where the highway continues as Massachusetts Route 32, north to NH 12 in Keene.', ' NH 32 connects the southern Cheshire County towns of Richmond and Swanzey with Keene and Athol, Massachusetts.']], ['New Hampshire Route 124', ['New Hampshire Route 124 (abbreviated NH 124) is a 28.083 mi east–west highway in southern New Hampshire, United States.', ' It runs from Marlborough to the Massachusetts border.']], ['Dublin Pond', ['Dublin Pond or Dublin Lake is a 236 acre water body located in Cheshire County in southwestern New Hampshire, United States, in the town of Dublin.', ' The pond lies at an elevation of 451 m above sea level, near the height of land between the Connecticut River/Long Island Sound watershed to the west and the Merrimack River/Gulf of Maine watershed to the east.', ' Water from Dublin Pond flows west through a series of lakes into Minnewawa Brook, a tributary of the Ashuelot River, which flows to the Connecticut River at Hinsdale, New Hampshire.', ' New Hampshire Route 101, a two-lane highway, runs along the northern shore of the lake, and the town center of Dublin is less than one mile to the east.']], ['New Hampshire Route 78', ['New Hampshire Route 78 (abbreviated NH 78) is a 3.456 mi secondary state highway in Cheshire County in the southern part of the U.S. state of New Hampshire.', ' A northward extension of Massachusetts Route 78, NH 78 runs entirely within the town of Winchester from the state border to downtown, where it ends at New Hampshire Route 10 and New Hampshire Route 119.']], ['West Chesterfield, New Hampshire', ['West Chesterfield is an unincorporated community in the town of Chesterfield in Cheshire County, New Hampshire, United States.', ' It is located north of New Hampshire Route 9 in a valley leading to the Connecticut River.', ' Via Route 9, Brattleboro, Vermont, is 3 mi to the west, and Keene, New Hampshire, is 14 mi to the east.']], ['East Jaffrey Historic District', ['The East Jaffrey Historic District is a historic district running roughly along NH Route 124 (Main Street) through Jaffrey, New Hampshire.', ' It encompasses what is now the economic and civic heart of the town, centered on the Jaffrey Mills and the crossing of the Contoocook River by Route 124.', \" It extends as far west as St. Patrick's Church beyond Charlonne Street and as far east as the US Post Office building at Route 124 and Ellison Street.\", ' To the north it extends along Peterborough Street (United States Route 202) to Christian Court, and to the south it extends along River Street (also US 202) and School Street to their junction.', ' The district includes early 19th-century residential structures, as well as industrial buildings and housing associated with the Jaffrey Mills which arose in the mid-19th century.', ' The area was known as \"East Jaffrey\" prior to its rise in economic ascendancy over what is now Jaffrey Center, the center of Jaffrey when it was chartered in 1773.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.613\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abbdd6955429931dba145b5', 'answer': 'Harry Booth', 'question': 'Who directed the 1971 film in which Pat Ashton starred in?', 'supporting_facts': [['Pat Ashton', 1], ['On the Buses (film)', 0]], 'context': [['Jenny Agutter', ['Jennifer Ann Agutter {\\'1\\': \", \\'2\\': \", \\'3\\': \\'OBE\\', \\'4\\': \"} (born 20 December 1952) is a British actress.', ' She began her career as a child actress in 1964\\'s \"East of Sudan\" and went on to appear in \"Star!', '\" and two adaptations of \"The Railway Children\"—the BBC\\'s 1968 television adaptation and the 1970 film version.', ' She also starred in the critically acclaimed 1971 film \"Walkabout\", before moving to Hollywood in 1974.', ' Her Hollywood film roles included parts in \"Logan\\'s Run\" (1976), \"An American Werewolf in London\" (1981) and \"Child\\'s Play 2\" (1990).', ' Agutter won the Emmy Award for Outstanding Supporting Actress in a Drama for the 1971 TV film \"The Snow Goose\", and the BAFTA Award for Best Actress in a Supporting Role for playing Jill Mason in the 1977 film \"Equus\".']], ['The Tales of Beatrix Potter (ballet)', [\"The Tales of Beatrix Potter is a 1992 ballet adapted for stage by Anthony Dowell from a 1971 film that was choreographed by Frederick Ashton that in turn was based on the children's books by Beatrix Potter.\"]], ['Pat Ashton', ['Pat Ashton (28 February 1931 – 23 June 2013) was an English actress.', ' Her engaging cockney, blonde persona is best remembered for appearances in English TV-sitcom film spin-offs \"On the Buses\" (1971) and \"Mutiny on the Buses\" (1972).']], ['Christian de Chalonge', ['Christian de Chalonge (born 21 January 1937) is a French film director and screenwriter.', ' He directed the 1971 film \"The Wedding Ring\", which starred Anna Karina.']], ['Lou Lombardo (filmmaker)', ['Lou Lombardo (February 15, 1932 – May 8, 2002) was an American filmmaker whose editing of the 1969 film \"The Wild Bunch\" has been called \"seminal\".', ' In all, Lombardo is credited on more than twenty-five feature films.', ' Noted mainly for his work as a film and television editor, he also worked as a cameraman, director, and producer.', ' In his obituary, Stephen Prince wrote, \"Lou Lombardo\\'s seminal contribution to the history of editing is his work on \"The Wild Bunch\" (1969), directed by Sam Peckinpah.', ' The complex montages of violence that Lombardo created for that film influenced generations of filmmakers and established the modern cinematic textbook for editing violent gun battles.\"', ' Several critics have remarked on the \"strange, elastic quality\" of time in the film, and have discerned the film\\'s influence in the work of directors John Woo, Quentin Tarantino, Kathryn Bigelow, and the Wachowskis, among others.', \" While Lombardo's collaboration with Peckinpah lasted just a few years, his career was intertwined with that of director Robert Altman for more than thirty years.\", ' Lombardo edited Altman\\'s 1971 film \"McCabe & Mrs. Miller\" (1971), which had \"a radical approach to the use of dialogue and indeed other sound, both in and beyond the frame.\"', ' Towards the end of his career Lombardo edited \"Moonstruck\" (1987) and two other films directed by Norman Jewison.', ' While his editing is now considered \"revolutionary\" and \"brilliant\", Lombardo was never nominated for editing awards during his career.']], ['On the Buses (film)', ['On the Buses is a 1971 British comedy film directed by Harry Booth and starring Reg Varney and Doris Hare.', ' The film is the first spin-off film from the TV sitcom \"On the Buses\" and was followed by two further films \"Mutiny on the Buses\" (1972) and \"Holiday on the Buses\" (1973).', ' The films are set within a slightly different canon from the TV series; Stan and Jack work for a different bus company (Town & District instead of Luxton & District), and the three films form a loose story arc where Arthur and Olive become parents (despite their apparently sexless marriage).']], ['Together (1971 film)', ['Together is a 1971 film directed by Sean S. Cunningham.', \" Cunningham's first film attracted Wes Craven who wanted to be in the film business.\", \" This was Craven's first credit.\", ' Cunningham and Craven would later work on \"The Last House on the Left\".', ' The film features a young Marilyn Chambers, billed under her real name, Marilyn Briggs, before she starred in \"Behind the Green Door\".']], ['Rainbow Bridge (film)', ['Rainbow Bridge is a 1971 film directed by Chuck Wein about different countercultural figures interacting on the Hawaiian island of Maui.', ' He described it as \"a kind of space-age \"Candid Camera\".', \" We're going to place Pat [New York model Pat Hartley, the protagonist] in all kinds of real-life situtations, and film what happens.\", ' We\\'re going to shoot a lot of film and just see what comes out of it.\"', ' Harry Shapiro adds, \"the idea was to shoot an antidote to \"Easy Rider\", showing the positive side of the youth movement.\"']], ['In the First Place', ['\"In the First Place\" is a song by the English rock group the Remo Four.', ' It was released as a single in January 1999 to accompany the re-release of the 1968 psychedelic film \"Wonderwall\", directed by Joe Massot.', ' The song was written by Colin Manley and Tony Ashton of the Remo Four and recorded in London in January 1968 during the sessions for George Harrison\\'s \"Wonderwall Music\" soundtrack album.', \" Having produced the track for the band, Harrison unearthed the recording 30 years later when supplying Massot with the master tapes for the film's music.\", \" Ashton and the Remo Four's drummer, Roy Dyke, also recorded the song with their subsequent group, Ashton, Gardner and Dyke, in 1969.\"]], ['Thick as Thieves (TV series)', ['Thick as Thieves is a British sitcom which was broadcast between 1 June and 20 July 1974 on LWT.', ' It was created and written by Dick Clement and Ian La Frenais.', ' There were 8 episodes over one series and starred Bob Hoskins, John Thaw and Pat Ashton.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 139/500 [00:00<00:00, 1388.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.614\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adf6a5c5542995ec70e8ff9', 'answer': 'Watertown, New York', 'question': 'In what city does the most successful American and international five-and-dime business have a historic building?', 'supporting_facts': [['F. W. Woolworth Building (Watertown, New York)', 0], ['F. W. Woolworth Company', 1]], 'context': [['F. W. Woolworth Company', [\"The F. W. Woolworth Company (often referred to as Woolworth's, or Woolworth) was a retail company and one of the original pioneers of the five-and-dime store.\", ' It was arguably the most successful American and international five-and-dime business, setting trends and creating the modern retail model which stores follow worldwide today.']], ['C.M. Sanborn Building', ['The C.M. Sanborn Building is a historic building located in Maquoketa, Iowa, United States.', ' Built in 1896, it is significant as an example of High Victorian Italianate architecture.', ' The three-story, brick building features cast hoodmolds above the windows, twin oriel windows, and an elaborate cornice.', ' C.M. Sanborn was a local grocer whose business operated out of a number of buildings in the central business district before he built this building.', ' He acted as the general contractor for the construction of this building, and hired two local builders to construct it.', ' William Hancock was a brick mason, and W.P. Thomas was a carpenter.', ' Sanborn filed for bankruptcy around 1911, and was forced to sell the building.', ' A variety of retail establishments have occupied the main floor, while the second floor was rented as office space.', ' A Masonic lodge occupied the third floor shortly after the building was completed, and remained until 1968.', ' The Masons owned the building by that time.', ' The building was listed on the National Register of Historic Places in 1991.']], ['Maynard Town Hall and Jail', ['Maynard Town Hall and Jail is a historic building located in Maynard, Iowa, United States.', ' Before this building was constructed the town had no building to enact its business.', \" They used the mayor's house or place of business for his office and council meetings.\", ' After it was completed, this building served as a jail into the late 1930s and as the city hall until 1952, when a new community building was completed.', ' For most of the time it served as a jail it was outfitted with bars and had bunks.', ' It was a short-term lock-up for those who committed minor infractions.', ' It was used for a public library beginning in the 1940s.', ' After its use as a city hall, it was bought by a local merchant for use in his grocery store in 1957.', ' The building was listed on the National Register of Historic Places in 1997.']], ['Pioneer Implement Company', ['The Pioneer Implement Company, also known as the International Harvester Transfer House, is a historic building located in Council Bluffs, Iowa, United States.', ' Eli Shugart, Ferdinand Weis and F.R. Davis formed Pioneer Implement Company in 1893, and they built the four-story Late Victorian section of the building the same year.', ' This was during a period of growth in the city\\'s \"Implement District,\" which is located to the south of the central business district.', ' What made this company standout from the others in the district is that it was locally owned and operated agricultural implement business, rather than one owned by a factory from the east.', ' The company went out of business in 1915 and the building was taken over by International Harvester, which owned the building next door.', ' They added the single story brick addition to the south in 1927.', ' When IH closed their Council Bluffs operation in 1964 they were the only farm implement that was still in business in the Implement District.', ' The building was occupied by various businesses after that, including United Parcel Service.', ' Artspace, Inc. acquired the building and converted it into live/work spaces for artists.', ' The building was listed on the National Register of Historic Places in 2008.']], ['Sioux City Linseed Oil Works', ['The Sioux City Linseed Oil Works is a historic building located in Sioux City, Iowa, United States.', ' It is located in an area east of the central business district where warehouses and other industrial buildings are located.', ' It housed the Sioux City Linseed Oil Works (initially known as Hubbard and Gere) from 1884, when it was built, until 1927 when its owner, the American Linseed Oil Company, ceased production here.', ' The building was largely destroyed in a fire that began late in the night of May 20, 1891.', ' Mankato, Minnesota architect William D. McLaughlin stepped in to complete the work begun by Sioux City architect E.W. Loft.', ' Essentially, the building was rebuilt and was very similar in appearance to the original structure.', ' Gone was the gable roof on the westernmost wing, which was replaced by a flat roof, and two floors were added to the rebuilt press room annex in the back.', ' The building was acquired by Bekins Van and Storage Company in 1928.', ' They began operations here two years later, and they remained the primary tenant in the building until 1972.', ' There was a variety of other tenants who were housed here over the years.', ' The building was listed on the National Register of Historic Places in 2008.']], ['Sandwich–Marseilles Manufacturing Building', ['The Sandwich–Marseilles Manufacturing Building, also known as the Dwarfies/Breeders Supply Building, is a historic building located in Council Bluffs, Iowa, United States.', ' This building was built by the Sandwich Manufacturing Company and the Marseilles Manufacturing Company in what is known as the Implement District, an industrial area south of the central business district that was home to farm implement manufacturers.', ' The two-story section on the north side was completed in 1883, and the single-story addition on the south side was completed sometime between 1889 and 1891.', ' Marseilles then occupied the addition while Sandwich remained in the original building.', \" There was a shift from agricultural implements to food processing as the area's business interests began to diversify.\", ' Dwarfies Corporation, a cereal manufacturer, took over the building in 1929.', ' It is the only remaining building left in the city to illustrate this shift.', ' The building was damaged in a fire in 1947.', ' Dwarfies rebuilt this building and then built a new factory in 1949 along U.S. Route 6.', ' Breeders Supply Company, an international mail order business for breeding supplies, moved into this building the following year.', ' They used it as a warehouse for ten years.', ' The building was listed on the National Register of Historic Places in 2014.']], ['First National Bank of Mason City', ['The First National Bank of Mason City, also known as Norwest Bank Building and City Center of Mason City, is a historic building located in Mason City, Iowa, United States.', ' It was designed by the Des Moines architectural firm of Liebbe, Nourse & Rasmussen, and it was the only Mason City commission for this firm.', ' Completed in 1911, it was constructed by C.E. Atkinson of Webster City, Iowa who had built several other H.F. Liebbe designs.', ' The 6½-story building follows the Early Commercial style.', ' It features modestly decorated main floor and attic level with five floors of rather plain brick construction in between.', ' The bank occupied most of the first floor and some of the office space above, while the other office space was taken up by professional offices.', ' By the 1960s the bank occupied the whole building.', ' John Dillinger, Baby Face Nelson, John Hamilton and Tommy Carroll robbed the bank on March 13,1934 and stole about $50,000.', ' Dillinger was wounded in an exchange of gunfire during the heist.', ' The building to the north of the bank was torn down in 1982 and a two-story annex to the bank replaced it.', ' The bank building was individually listed on the National Register of Historic Places in 1997, and as a contributing property in the Mason City Downtown Historic District in 2005.', ' City Center of Mason City Inc. bought the building in 1995, and remodeled it into apartments and offices.']], ['Hamilton Brothers Building', ['The Hamilton Brothers Building, also known as Warfield-Pratt & Howell Co.', ' Wholesale Grocers, and the Hach Brothers Company, is a historic building located in Cedar Rapids, Iowa, United States.', ' This is the only extant building associated with local businessman and politician John Taylor Hamilton.', ' He was a representative of Cyrus McCormick at the time his company was opening new markets for his mechanical reaper and other implements.', ' Hamilton expanded his business to other implement manufactures and included product lines for urban dwellers as well.', ' His business grew beyond the local area to include the entire state.', ' Built as a warehouse in 1899, it is the only building of this type left near the central business district.', ' It is also the largest building of this type in the city.', ' The four-story, brick, Romanesque Revival sturcture features large round arch openings on the main floor and smaller windows on the upper floors.', ' It was listed on the National Register of Historic Places in 1994.']], ['Evans Block (Sioux City, Iowa)', ['The Evans Block, also known as Northwestern National Bank Building, is a historic building located in Sioux City, Iowa, United States.', ' The city experienced a building boom that began in the late 1880s and continued into the early 1890s.', ' Fred T. Evans, an entrepreneur who had business interests in Iowa, Nebraska and South Dakota, had this building constructed to house Northwestern National Bank of which he was the president.', ' The bank occupied the main level and other offices were housed on the upper floors.', ' Local architect Charles Brown designed the four-story Romanesque Revival style building.', \" The Black Hills sandstone for the public facades was from Evans' quarry.\", \" The Panic of 1893 brought Sioux City's building boom to an end, and the Evans block was sold in January 1895.\", ' Subsequently, the building has housed a hotel, a factory, a saloon, and a variety of stores.', ' It was individually listed on the National Register of Historic Places in 1985, and as a contributing property in the Fourth Street Historic District in 1995.']], ['F. W. Woolworth Building (Watertown, New York)', ['The Woolworth Building is an historic building in Watertown, New York.', ' It is a contributing building in the Public Square Historic District.', \" Plans for the Woolworth Building were begun in 1916 by Frank W. Woolworth, the founder of the Woolworth's chain of department stores.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.615\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8ced46554299441c6b9f82', 'answer': 'June 13, 1960', 'question': 'When was the father of the professional ice hockey player with the middle name \"Fosgaard\" born?', 'supporting_facts': [['Olaf Eller', 0], ['Olaf Eller', 2], ['Lars Eller', 0]], 'context': [['Patrick Eaves', ['Patrick Campbell Eaves (born May 1, 1984) is a Canadian-American professional ice hockey forward, who is currently playing for the Anaheim Ducks of the National Hockey League (NHL).', ' Born in Calgary, Alberta, and raised in Faribault, Minnesota, Eaves holds Canadian and American citizenship and has represented the United States in international ice hockey tournaments.', ' He is the son of retired ice hockey player Mike Eaves, and brother of retired ice hockey player Ben Eaves.']], ['Anders Gozzi', ['Anders Gozzi (born (1967--) 12, 1967 ) is a Swedish former professional ice hockey player and currently the general manager of the AIK IF organization.', ' In his career as a professional ice hockey player he played for AIK, Brynäs IF, and Düsseldorfer EG.', ' In his first season with AIK, in the 1986–87 season, the team became promoted to Elitserien.', ' He played in AIK during the majority of his career, and scored 315 points in 579 Elitserien (SEL) games.', ' He became Elitserien champions with Brynäs IF in the 1992–93 season.', ' He ended his ice hockey player career with AIK in the 2003–04 season, when the team played in HockeyAllsvenskan.', ' He also was the general manager of AIK that season, and in the 2004–05 season he also became an assistant coach, replacing Tomas Winje mid-season.', ' In the 2007–08 season he was the head coach of AIK from early December 2007 until the end of the season.', ' Since the end of the 2007–08 season, he has been the general manager of AIK.']], ['Mikael Lindholm', ['Mikael Lindholm (born 19 December 1964 in Gävle, Sweden) is a former professional ice hockey player who played for the Los Angeles Kings in the National Hockey League.', ' His son Elias Lindholm is a professional ice hockey player and was selected by the Carolina Hurricanes in the 1st round (5th overall) of the 2013 NHL Entry Draft.', '.', ' Mikael is the father of the NHL hockey player, Elias Lindholm and the uncle of Calle Järnkrok.']], ['L. S. Dukowski', ['Wladislaw Laudas Jozef \"Duke\" Dukowski (August 31, 1900 – September 26, 1976) was a professional ice hockey player who played 206 games in the National Hockey League.', ' He played defense for the New York Rangers, New York Americans, and Chicago Black Hawks.', ' His middle initial is sometimes erroneously stated as \"S\" when in fact his middle name was Joseph.']], ['Brian Smith (ice hockey, born 1940)', ['Brian Desmond \"Smitty\" Smith (September 6, 1940 – August 2, 1995) was a Canadian professional hockey player and sportscaster.', ' Smith was born in Ottawa, Ontario, the son of former professional ice hockey player Des Smith and brother of former professional hockey goaltender Gary Smith.', ' Smith was a professional ice hockey player from 1960 to 1973, playing in the National Hockey League (NHL) with the Los Angeles Kings and Minnesota North Stars.', ' Following his hockey career, Smith was a broadcaster for CJOH-TV in Ottawa until 1995, when he was shot and killed by gunman Jeffrey Arenburg.']], ['Lars Eller', ['Lars Fosgaard Eller (born 8 May 1989) is a Danish professional ice hockey player currently playing for the Washington Capitals of the National Hockey League (NHL).', ' He was drafted by the St. Louis Blues in the first round, 13th overall in the 2007 NHL Entry Draft.']], ['Smokey Harris', ['Thomas Wilfred \"Smokey, Fred\" Harris (October 11, 1890 – June 4, 1974) was a Canadian professional ice hockey player.', ' Harris played in the Pacific Coast Hockey Association (PCHA), the National Hockey League (NHL) and the Western Canada Hockey League (WCHL).', ' Harris was born in Port Arthur, Ontario.', ' His brother Henry was also a professional ice hockey player.', \" Harris scored the first goal in Boston Bruins' franchise history.\"]], ['Nathan Walker', ['Nathan Walker (born 7 February 1994) is an Australian professional ice hockey player currently playing for the Hershey Bears of the American Hockey League (AHL) and a prospect for the Washington Capitals of the National Hockey League.', ' Walker, who was born in Wales, grew up in Australia and first played ice hockey there.', ' He moved to the Czech Republic in 2007 in order to further his career, and joined the junior program of HC Vítkovice, a member of the Czech Extraliga.', ' He first played for the senior team in 2011, becoming the first Australian ice hockey player to play for a professional senior team in Europe.', ' Along with HC Vítkovice, Walker was loaned to several lower-level Czech teams.']], ['Olaf Eller', ['Olaf Eller (born June 13, 1960) is a Danish ice hockey coach.', \" He is currently the head coach of Denmark men's national junior ice hockey team.\", ' His son is Lars Eller, who was drafted 13th overall to the St. Louis Blues in 2007 and currently plays for the Washington Capitals.', ' His son Mads Eller won the Memorial Cup with the Edmonton Oil Kings of the Western Hockey League, and currently plays for the Gentofte Stars in the Danish Metal Ligaen.']], ['Tim Cranston', ['Tim Cranston (born 13 December 1962 in Halifax, Nova Scotia) is a retired professional ice hockey player who holds dual Canadian and British nationality.', ' He played in Europe between 1985 and 1999 except for one game in the 1986–87 season played in the American Hockey League.', ' He was also a member of the Great Britain national ice hockey team between 1993 and 1997.', \" Whilst living in the United Kingdom, Cranston was the founding chairman of the British Ice Hockey Player's Association (GB).\", ' Currently living in Halifax, Nova Scotia, Cranston is working as a sports agent and lawyer for the sports and entertainment industries.', ' He was inducted into the British Ice Hockey Hall of Fame in 2010.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.615\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac3dfee554299204fd21eab', 'answer': 'Love Hina', 'question': 'The man known as the voice of \"Elgar\" voices the English character in what Manga inspired by Keitaro Arima?', 'supporting_facts': [['Keitarō Urashima', 0], ['Keitarō Urashima', 1], ['Keitarō Urashima', 2], ['Derek Stephen Prince', 0]], 'context': [['The Dream of Gerontius', ['The Dream of Gerontius, Op. 38, is a work for voices and orchestra in two parts composed by Sir Edward Elgar in 1900, to text from the poem by John Henry Newman.', \" It relates the journey of a pious man's soul from his deathbed to his judgment before God and settling into Purgatory.\", ' Elgar disapproved of the use of the term \"oratorio\" for the work, though his wishes are not always followed.', \" The piece is widely regarded as Elgar's finest choral work, and some consider it his masterpiece.\"]], ['Naru Narusegawa', ['Naru Narusegawa (成瀬川 なる , Narusegawa Naru ) is a fictional character in the \"Love Hina\" series by Ken Akamatsu and one of the central characters in the franchise.', ' Known for her fiery temper and tendency to use physical violence to punish central character Keitaro Urashima, she is the first female introduced in the series who is trying to pass the University of Tokyo entrance exams.', ' Her studies along with her developing relationship with Keitaro is one of the focal points in the series.', ' Her Japanese voice actress is Yui Horie and her English voice actor is Dorothy Elias-Fahn.']], ['Prefectural Earth Defense Force', ['Prefectural Earth Defense Force (県立地球防衛軍 , Kenritsu Chikyū Bōei Gun ) is a manga series by Kōichirō Yasunaga which ran in Shōnen Sunday Super beginning in 1983.', ' The manga was written as a parody gag manga inspired by the tokusatsu series \"Ultra Seven\".', ' An anime OVA based on the manga was released in 1986.', ' The anime was released on DVD in North America by ADV Films on April 18, 2006.']], ['Shamo (manga)', ['Shamo (軍鶏 ) is a Japanese action manga series written by Izo Hashimoto and illustrated by Akio Tanaka.', ' It was started to be serialized in \"Weekly Manga Action\" in 1998 and moved to \"Evening\" in 2004.', ' It was discontinued in 2007 due to creative differences but returned in 2011 and ended in 2015.', ' It tells a story of a boy who killed his parents and turned himself into a cold-blooded martial artist.', ' The manga inspired a Hong Kong film adaptation that was released in 2007.']], ['List of Love Hina episodes', ['Love Hina (ラブ ひな , Rabu Hina ) is a Japanese manga series written and illustrated by Ken Akamatsu.', ' It was adapted into a 24 episode anime series by Xebec, which aired in Japan on TV Tokyo from April 19, 2000 to September 27, 2000.', ' The opening theme was \"Sakura Saku\" (サクラサク ) and the closing theme was \"Kimi Sae Ireba\" (君さえいれば ) .', ' Both songs were written by Ritsuko Okazaki and performed by Megumi Hayashibara.', ' The two themes were released as a CD single, which debuted on the Oricon charts at Number 7.', ' A 25th episode was later created and released as a DVD bonus.', ' The series and bonus episode were directed by Yoshiaki Iwasaki, written by Shō Aikawa and featured character designs by Makoto Uno.', ' After the series finished, Christmas and Spring specials followed, and finally a 3 part original video animation (OVA) series called \"Love Hina Again\".', ' The series follows the daily life of Keitaro Urashima, the manager of an all-girls dorm, as he attempts to pass the Tokyo University entrance exams and to find the girl he promised to enter Tokyo U with when he was a child.']], ['List of GetBackers episodes', ['The episodes for the anime series \"GetBackers\" were produced by Studio Deen and based on the manga series of the same name written by Yuya Aoki and illustrated by Rando Ayamine.', ' The series premiered on Tokyo Broadcasting System in Japan on October 5, 2002 and ran for forty-nine episodes until September 20, 2003 under direction of Kazuhiro Furuhashi and Keitaro Motonaga.', ' The plot follows the \"GetBackers\", a group that retrieves anything that was lost.', ' The team is primary composed by Ban Mido, a man born with the illusionary technique \"Evil Eye\", and Ginji Amano the former leader of a gang called \"The VOLTS\", a powerful group in the dangerous territory called the Limitless Fortress in Shinjuku.']], ['Derek Stephen Prince', ['Derek Stephen Prince (born February 5, 1969 in Inglewood, California) is an American voice actor who is most memorable for his various roles in the \"Digimon\" series, as well as the voice of Elgar in the live-action \"Power Rangers Turbo\" and \"Power Rangers in Space\".']], ['William Henry Reed', ['William Henry \"Billy\" Reed (29 July 18752 July 1942) was an English violinist, teacher, minor composer, conductor and biographer of Sir Edward Elgar.', ' He was leader of the London Symphony Orchestra for 23 years (1912–1935), but is best known for his long personal friendship with Elgar (1910–1934) and his book \"Elgar As I Knew Him\" (1936), in which he goes into great detail about the genesis of the Violin Concerto in B minor.', \" The book also provides a large number of Elgar's sketches for his unfinished Third Symphony, which proved invaluable sixty years later when Anthony Payne elaborated and essentially completed the work, although Reed wrote that in his view the symphony could not be completed.\"]], ['Keitarō Urashima', ['Keitarō Urashima (浦島 景太郎 , Urashima Keitarō ) is a fictional character and the protagonist from Ken Akamatsu\\'s manga and anime \"Love Hina\".', ' He is voiced by Yūji Ueda (Japanese) and Derek Stephen Prince (English).', ' His name is inspired by Keitarō Arima as well as the mythological character Urashima Tarō']], ['Akaneiro ni Somaru Saka', ['Akaneiro ni Somaru Saka (あかね色に染まる坂 , lit.', ' \"The Hill Dyed Rose Madder\") , also known in short as \"Akasaka\", is a Japanese adult visual novel developed by Feng and first released for the PC as a DVD on July 27, 2007.', ' A version without adult content was released under the title \"Akaneiro ni Somaru Saka: Parallel\" on July 31, 2008 by GN Software for the PlayStation 2.', ' A port of this version of the game was released for the PlayStation Portable on December 17, 2009 under the title \"Akaneiro ni Somaru Saka: Portable\".', ' The gameplay in \"Akaneiro ni Somaru Saka\" follows a plot line which offers pre-determined scenarios with courses of interaction, and focuses on the appeal of the six female main characters.', ' Two light novels were produced in December 2007 and February 2008 written by different authors, and an Internet radio show began in April 2008.', ' A manga adaptation began serialization in Kadokawa Shoten\\'s seinen magazine \"Comp Ace\" on June 26, 2008 illustrated by Homare Sakazuki.', ' An anime adaptation produced by TNK and directed by Keitaro Motonaga aired in Japan between October and December 2008.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.616\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae8167b55429952e35eaa06', 'answer': 'Battle of the Rosebud', 'question': 'What 1876 battle featured the Other Magpie?', 'supporting_facts': [['The Other Magpie', 0], ['Battle of the Rosebud', 0]], 'context': [['Battle of Congaree Creek', ['The Battle of Congaree Creek was a four-hour action that took place in the waning days of the American Civil War, fought in Lexington County, South Carolina, on February 15, 1865, just south of Columbia.', \" The battle featured the Union's Army of the Tennessee against the Confederacy's Army of Tennessee meeting at a half-mile-long earthwork erected by Southern forces near the Old State Road Bridge over Congaree Creek.\", \" Confederate General George Dibrell's dismounted cavalry brigade, supported by infantry and artillery, manned the earthworks, but General Charles Woods' 1st Division of General John A. Logan's XV Corps pushed skirmishers ahead while one of Woods' brigades crossed upstream and turned the Southerners' right flank.\", \" Dibrell's force withdrew from Congaree Creek and then from its earthworks, retreating to Columbia.\", ' Though the Confederates set fire to the bridge, the Federals saved it and made their camp nearby that night, according to a historical marker erected at the site by the 15th Regiment S.C. Volunteer Camp of the Sons of Confederate Veterans.']], ['Battle of the Rosebud', ['The Battle of the Rosebud (also known as the Battle of Rosebud Creek) occurred June 17, 1876, in the Montana Territory between the United States Army and its Crow and Shoshoni allies against a force consisting mostly of Lakota Sioux and Northern Cheyenne Indians during the Great Sioux War of 1876.', ' The Cheyenne called it the Battle Where the Girl Saved Her Brother, because of an incident during the fight involving Buffalo Calf Road Woman.', \" General George Crook's offensive was stymied by the Indians, led by Crazy Horse, and he awaited reinforcements before resuming the campaign in August.\"]], ['Battle of Olongapo', ['The Battle of Olongapo was fought September 18–23, 1899, during the Philippine–American War.', ' The battle featured both land and sea fighting, of which the objective was the destruction of the single Filipino artillery gun in Olongapo, a menace to American ships crossing the nearby sea.']], ['Naval Battle of Campeche', ['The Naval Battle of Campeche took place on April 30, 1843, and May 16, 1843.', ' The battle featured the most advanced warships of its day, including the Mexican steamer \"Guadalupe\" and the equally formidable \"Moctezuma\" which engaged a squadron of vessels from the Republic of Yucatan and the Republic of Texas.', ' The latter force consisted of the Texas Navy flagship sloop-of-war \"Austin\", commanded by Commodore Edwin Ward Moore, the brig \"Wharton\", and several schooners and five gunboats from the Republic of Yucatán, commanded by former Texas Navy Captain James D. Boylan.', ' Texas had declared its independence in 1836 but by 1843 Mexico had refused to recognize it.', ' In Yucatán, a similar rebellion had begun and was fought off-and-on from 1836 to 1846.', ' The battle ended in a combined Yucatecan and Texan victory.']], ['Lame Deer', ['Lame Deer (died 1877) (Miniconjou Lakota), was a Wakpokinyan band leader (vice chief).', ' This group of Lakota were opposed to agreeing to the 1868 Treaty of Fort Laramie, which required the Lakota to cede much of their territory to the United States.', ' He was present at the 1876 Battle of the Greasy Grass, also known as the Battle of the Little Bighorn, where the combined Lakota and allied forces dealt an overwhelming defeat to United States forces.']], ['Battle of Ezra Church', ['The Battle of Ezra Church, also known as the Battle of Ezra Chapel and the Battle of the Poor House was fought on July 28, 1864, in Fulton County, Georgia, during the American Civil War.', \" Part of the Atlanta Campaign, the battle featured Maj. Gen. William T. Sherman's Union Army of the Tennessee against the Army of Tennessee, commanded by Lt. Gen. John B. Hood, which was defending the Confederate stronghold of Atlanta, Georgia.\"]], ['Battle of Kuwait International Airport', ['The Battle of Kuwait International Airport occurred on February 27, 1991 during the 1st Gulf War.', \" It was a tank battle between the United States and Ba'athist Iraq.\", ' Despite being a very large battle it is often overlooked compared to the other battles which took place during the war.', ' No less than elements of 18 divisions total participated in this battle.', ' U.S. Army Special Forces units and multiple Iraqi Commando units were also in theatre.', ' In reality the battle took place over a span of three days despite the primary battle at Kuwait International Airport lasting only one day.', ' Much of the combat actually took place en route to the airport.', ' The battle featured the \"Reveille Engagement\" which went on to become the biggest and fastest tank battle in United States Marine Corps\\' entire history.']], ['Northern Cheyenne Indian Reservation', ['The Northern Cheyenne Indian Reservation (Tsėhéstáno in Cheyenne, formerly named the Tongue River Indian Reservation) is home of the federally recognized Northern Cheyenne Tribe.', ' Located in southeastern Montana, the reservation is approximately 444,000 acres in size and home to approximately 5,000 Cheyenne people.', ' The tribal and government headquarters are in Lame Deer, which is also the home of the annual Northern Cheyenne Pow wow.', ' The reservation is bounded on the east by the Tongue River and on the west by the Crow Reservation.', ' There are small parcels of non-contiguous off-reservation trust lands in Meade County, South Dakota, northeast of the city of Sturgis.', ' Its timbered ridges that extend into northwestern South Dakota are part of Custer National Forest and it is approximately 40 mi east of the site of the 1876 Battle of the Greasy Grass (known to most Americans as the Battle of the Little Bighorn).']], ['The Other Magpie', ['The Other Magpie was a Crow woman who fought in the Battle of the Rosebud on the side of General Crook against the Sioux and Cheyenne.', ' Pretty Shield, a Crow author and medicine woman, described her as being wild and attractive, but not having a man.', ' She fought because her brother had recently been killed by the Sioux and she sought revenge against them.', ' Most of the Crow carried rifles, but The Other Magpie carried only her belt knife and her coup stick.', ' She counted coup on a Sioux warrior and eventually killed and scalped him.', ' The scalp that she took was one of only eleven taken in the battle.', ' Pretty Shield described her as having tied a feather on the end of her coup stick to symbolize her achievement.', ' Later, she cut the scalp into pieces and gave them to the male warriors so they would have more scalps for the dance after the battle.']], ['Second Battle of Caloocan', ['The Second Battle of Caloocan, alternately called the Second Battle of Manila, was fought from February 22 to 24, 1899, in Caloocan during the Philippine–American War.', ' The battle featured a Filipino counterattack aimed at gaining Manila from the Americans.', ' This counterattack failed to regain Manila mainly because of lack of coordination among Filipino units and lack of artillery support.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.617\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a728b1c5542992359bc30e0', 'answer': 'Jonghyun', 'question': 'Which member of the boy group Shinee released their first studio album \"She is\"?', 'supporting_facts': [['She Is', 0], ['Kim Jong-hyun (singer)', 1]], 'context': [['Romeo (EP)', ['Romeo (Korean: 로미오 ) is the second EP of South Korean boy group Shinee.', ' It was released on May 25, 2009 in South Korea under the seal of the label S.M. Entertainment.', ' The EP consists of six tracks including the title song \"Juliette\" and is Shinee\\'s first Korean release after nine months hiatus.', ' On August 29, 2011 a Japanese version of \"Juliette\" was released as Shinee\\'s second Japanese single with the original Japanese song \"Kiss Kiss Kiss\" as a B-side.', ' The release peaked at #3 on the weekly Oricon chart.']], ['List of awards and nominations received by Shinee', ['South Korean boy group Shinee have received several awards and nominations for their music work.', ' The group was formed by S.M. Entertainment in 2008 and released their first full-length album, \"The Shinee World\", on August 28, 2008, which won the Newcomer Album of the Year at the 23rd Golden Disk Awards.', ' The first single released from the album was \"Sanso Gateun Neo (Love Like Oxygen)\" and won first place on \"M Countdown\" on September 18, 2008 making it the group\\'s first win on Korean music shows since debut.', ' Their second album \"Lucifer\" (2010) produced two singles, \"Lucifer\" and \"Hello\".', ' For their outstanding choreography the group was nominated for the Best Dance Performance Award at the Mnet Asian Music Awards in 2010. \"', 'Lucifer\" also won the Disk Bonsang Award at the 25th Golden Disk Awards as well as the Popularity Award.', ' On March 21, 2012 the group released their fourth EP \"Sherlock\" for which the group was awarded another Disk Bonsang Award at the 27th Golden Disc Awards and the Bonsang Award at the 22nd Seoul Music Award.', ' Also following the success of the lead single it was also nominated for Song of the Year at the 2012 Mnet Asian Music Awards.']], ['1 of 1 (album)', ['1 of 1 is the fifth Korean studio album and the eighth overall by South Korean boy band Shinee.', ' It was released digitally and physically on October 5, 2016, under S.M. Entertainment and distributed by KT Music.', ' The album contains nine songs, including the title track of the same name, \"1 of 1\".', ' Musically, the album is a modernized twist on the retro genre, and stretches back to the 1980–1990 period.', \" Additionally, based on their '90s theme, the group released a limited edition of cassette tapes besides the usual CD version.\", ' In order to promote the album, Shinee appeared on several South Korean music programs, such as \"Music Bank\", \"Show!', ' Music Core\", and \"Inkigayo\", where they performed material from the album.', ' On November 15, 2016, Shinee released a repackaged version of their fifth studio album titled \"1 and 1\" with 5 new songs, including the title track \"Tell Me What to Do\".']], ['Kim Jong-hyun (singer)', ['Kim Jong-hyun (born April 8, 1990), better known by the mononym Jonghyun, is a South Korean singer-songwriter, and radio host.', \" He is a vocalist of the South Korean boy group Shinee, and has further participated in S.M. Entertainment's project group S.M.\", ' The Ballad.', ' Jonghyun debuted as a solo artist on January 12, 2015, with his first EP, titled \"Base\".', ' In the same year, on September 17, Jonghyun released a compilation album, \"Story Op.1\".', ' On May 24, 2016, Jonghyun released his first studio album, \"She Is\", followed by his second compilation album, \"Story Op.2\" on April 24, 2017.']], ['She Is', ['She Is (Hangul: 좋아 ; RR: \"Joh ah \" \"good\") is the first studio album by South Korean singer-songwriter Jonghyun, released on May 24, 2016 by S.M. Entertainment and distributed by KT Music.']], ['Shinee World 2012', ['Shinee World 2012 (promoted as THE FIRST JAPAN ARENA TOUR \"SHINee WORLD 2012\") is the first Japan nationwide concert tour by South Korean boy group Shinee to support their first Japanese studio album, \"The First\".', ' The tour kicked off in Fukuoka on April 25, 2012 and ended in Hiroshima on July 1, 2012 with a total of 20 concerts in 7 cities.']], ['List of songs written by Kim Jong-hyun', ['Kim Jong-hyun (most often credited as Jonghyun), is a South Korean singer-songwriter and producer.', ' He began his musical career in 2008 as a member of the group Shinee and later formed the ballad group S.M.', ' The Ballad.', ' Jonghyun debuted as a composer happened to write Korean lyrics for the Shinees promotional single \"Juliette\", which was featured in the mini-album \"Romeo\", released in May 2009.', ' Participate in the writing of three songs on Shinee\\'s second Korean studio album, \"Lucifer\", the first, \"Up & Down\", was co-written with Misfit with the rap being written by Minho, the second, \"Obsession\", was completely written by Jonghyun with Minho once again working on his own rap, and the third \"Shout Out\" co-written by all members of Shinee, JQ and Misfit.', ' In 2012, Jonghyun co-wrote the lyrics to the song with \"Alarm Clock\" with Minho, a song about wishing to wake up from the nightmare of a past break up, and wrote the lyrics to \"Honesty\" which was described as a song written for the fans who had stayed by their side with unchanging love until that point.', ' Both songs were featured on the mini album, \"Sherlock\".']], ['Sherlock (EP)', ['\"Sherlock\" is the fourth EP of South Korean boy group Shinee.', ' The EP consists of seven tracks including the title song \"Sherlock (Clue + Note)\" a hybrid remix of the two songs.', ' It was released on March 21, 2012, in South Korea under the seal of the label S.M. Entertainment and distributed by KT Music.', ' The album was made available online worldwide on March 19, 2012.', \" The EP is Shinee's first Korean release after a year and 6 months hiatus.\"]], ['Shinee World 2013', ['Shinee World 2013 (promoted as JAPAN ARENA TOUR SHINee WORLD 2013 ～Boys Meet U～) is the second Japan nationwide concert tour by South Korean boy group Shinee to promote their second Japanese studio album, \"Boys Meet U\".', ' The tour kicked off in Saitama on June 28, 2013 and ended in Nagoya on December 11, 2013 with a total of 15 concerts in 9 cities.']], ['The First (album)', ['\"The First\" is the first Japanese studio album by South Korean boy group Shinee.', ' The album was scheduled for release on November 23, 2011, however it was delayed to December 7, 2011 in Japan under EMI Music Japan.', ' The album features three previously released singles, \"Replay\", \"Juliette\" and \"Lucifer\", all of which have ranked within the top three on Oricon charts.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.618\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8eed625542995085b374b9', 'answer': 'Westminster system', 'question': 'Which system of parliament was modeled after the United Kingdom and is also used in Canada? ', 'supporting_facts': [['Constitution of Alberta', 0], ['Constitution of Alberta', 1], ['Westminster system', 0]], 'context': [['Governance of England', ['There has not been a government of England since 1707 when the Kingdom of England ceased to exist as a sovereign state, as it merged with the Kingdom of Scotland to form the Kingdom of Great Britain.', ' Kingdom of Great Britain continued from 1707 until 1801 when it merged with the Kingdom of Ireland to form the United Kingdom of Great Britain and Ireland, which itself became the United Kingdom of Great Britain and Northern Ireland (UK) in 1922 (in reality; in name in 1927) upon independence for most of the island of Ireland.', ' The UK since then has gone through significant change to its system of government, with devolved parliaments, assemblies and governments in Scotland, Wales and Northern Ireland.', ' England, however, remains under the full jurisdiction, on all matters, of the Parliament of the United Kingdom of Great Britain and Northern Ireland and the UK government as no devolved administration has been created for England within the new structure.', ' This situation has led to the anomaly, known as the West Lothian question, which is that Scottish Members of Parliament (MPs) are able to vote on legislation that affects only England whereas English MPs can not vote on certain Scottish matters due to devolution.', ' In some cases, such as top-up university tuition fees and foundation hospitals, the votes of Scottish MPs have been crucial in helping pass legislation for England that the majority of English MPs have opposed.']], ['Constitution of Alberta', ['The Constitution of Alberta describes the fundamental rules under which the Canadian province of Alberta is governed.', \" As is typical of all Canadian provinces, and Westminster systems more generally, Alberta's is an unwritten constitution.\", \" Alberta's constitution, like Britain's (on which it is modeled), includes any and all pieces of legislation, court decisions, proclamations, and conventions which together inform how the province operates.\", ' Many statutes are important to understanding the governance of the province, but nowhere are they consolidated into a single document or even a list.', ' The office of Attorney-General at one time suggested 23 acts which might be included, but cautioned that this was not a \"definitive list\".', ' However, since Alberta is a part of federation, its powers are clearly delineated in law, via the Constitution of Canada.']], ['Australian Public Service', ['The Australian Public Service (APS) is the federal civil service of the Commonwealth of Australia responsible for the public administration, public policy, and public services of the departments and executive and statutory agencies of the Government of Australia.', \" The Australian Public Service was established at the Federation of Australia in 1901 as the Commonwealth Public Service and modeled on the Westminster system and United Kingdom's Civil Service.\", ' The establishment and operation of the Australian Public Service is governed by the \"Public Service Act 1999\" of the Parliament of Australia as an \"apolitical public service that is efficient and effective in serving the Government, the Parliament and the Australian public\".', ' The conduct of Australian public servants is also governed by a Code of Conduct and guided by the APS Values set by the Australian Public Service Commission.']], ['Irish measure', ['Irish measure or plantation measure was a system of units of land measurement used in Ireland from the 16th century plantations until the 19th century, with residual use into the 20th century.', ' The units were based on \"English measure\" but used a linear perch measuring 7 yard as opposed to the English rod of 5.5 yard .', ' Thus, linear units such as the furlong and mile, which were defined in terms of perches, were longer by a factor of 14:11 (~27% more) in Irish measure, while areas such as the rood or acre were larger by 196:121 (~62% more).', ' After the Act of Union 1800, Ireland was part of the United Kingdom of Great Britain and Ireland, whose Parliament passed the Weights and Measures Act 1824, which established English measure in Ireland as \"Imperial measure\" or \"statute measure\".', ' Imperial measure soon replaced Irish measure in the use of the Dublin Castle administration, but Irish measure persisted in local government, and longer still in private use.', ' A third system, \"Scotch measure\" or \"Cunningham measure\", was also used in Ulster Scots areas.']], ['Atlantis Music Prize', ['The Atlantis Music Prize is a music award annually given to the best full-length album from Newfoundland and Labrador, Canada, based only on artistic merit, regardless of genre, sales or record label.', ' The award, established in 2008 by St. John\\'s-based alternative newspaper \"The Scope\", includes a certificate prize of $1000.', ' The award is modeled after the Polaris Music Prize for all of Canada (which in turn is modeled after the Mercury Music Prize in the United Kingdom).']], ['Canada under British rule', ['Canada first came under British rule with the Treaty of Paris (1763) which ceded New France, of which Canada was a part, to the British Empire.', ' Gradually, other territories, colonies, and provinces that were part of British North America would be added to Canada.', ' The Royal Proclamation of 1763 enlarged the colony of Canada under the name of the Province of Quebec, which with the Constitutional Act 1791 became known as The Canadas.', ' With the Act of Union 1840 Upper and Lower Canada were joined to become the United Province of Canada.', ' Later, with Confederation in 1867, the British maritime colonies of New Brunswick and Nova Scotia were joined with the British colony of Canada to form the Dominion of Canada, which was subsequently divided into four provinces, Ontario, Quebec, New Brunswick, and Nova Scotia.', \" A number of other British colonies, such as Newfoundland and British Columbia, and large territories such as Rupert's Land initially remained outside of the newly formed federation.\", ' Over time, the remaining colonies and territories within British North America came under the control of Canada until the current geographic extent of the country was reached when Newfoundland and Labrador joined Canada in 1949.', ' Although confederation in 1867 led to an enlarged Dominion with increased autonomy over domestic affairs, Canada still remained a colony within the British Empire and was thus subordinate to the British Parliament until the enactment of the Statute of Westminster in 1931.', ' This statute recognized Canada as an independent peer coequal with the United Kingdom, and thus provided the Parliament of Canada with legislative sovereignty over all federal matters except the power to change the constitutional laws of Canada which remained under the purview of the Parliament of the United Kingdom.', \" Canada's final vestige of legal dependence on the United Kingdom was terminated in 1982 with the enactment of the Canada Act, subsequently providing Canada with full legal sovereignty completely independent of the United Kingdom.\"]], ['General Assembly House', ['The General Assembly House, colloquially called Shedifice by the members of parliament, was the first house of the New Zealand Parliament in Auckland.', ' It was in use by Parliament from 1854 until 1864 during the time that Auckland was the capital of New Zealand.', ' It was also used by the Auckland Provincial Council, with Auckland Province owning the building from 1858.', \" After the abolition of the provincial government system, the building was used by government's survey department and was then used by Auckland University College.\", ' The General Assembly House was demolished in 1917 to make way for Anzac Avenue.', ' Today, a reserve adjacent to Parliament Street commemorates the location where New Zealand Parliament met initially.']], ['Politics of Canada', ['The politics of Canada function within a framework of parliamentary democracy and a federal system of parliamentary government with strong democratic traditions.', ' Canada is a constitutional monarchy, in which the Monarch is head of state.', \" The country has a multi-party system in which many of its legislative practices derive from the unwritten conventions of and precedents set by the United Kingdom's Westminster Parliament.\", ' However, Canada has evolved variations: party discipline in Canada is stronger than in the United Kingdom and more parliamentary votes are considered motions of confidence, which tends to diminish the role of non-Cabinet Members of Parliament, (MPs).', ' Such members, in the government caucus, and junior or lower-profile members of opposition caucuses, are known as backbenchers. Backbenchers can, however, exert their influence by sitting in parliamentary committees, like the Public Accounts Committee or the National-Defence Committee.']], ['Westminster system', ['The Westminster system is a parliamentary system of government modelled after that which developed in the United Kingdom.', ' This term comes from the Palace of Westminster, the seat of the British parliament.']], ['British North America Acts', ['The British North America Acts 1867–1975 are the original names of a series of Acts at the core of the constitution of Canada.', ' They were enacted by the Parliament of the United Kingdom and the Parliament of Canada.', ' In Canada, some of the Acts were amended or repealed by the Constitution Act, 1982.', ' The rest were renamed in Canada as the \"Constitution Acts\".', ' In the United Kingdom, those Acts that were passed by the British Parliament remain under their original names.', ' The term \"British North America\" (BNA) refers to the British colonies in North America.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.618\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5addda9b5542992200553b5b', 'answer': 'Isla de Xativa', 'question': 'What was the island, on which Marinelli Glacier is located, formerly known as?', 'supporting_facts': [['Marinelli Glacier', 0], ['Isla Grande de Tierra del Fuego', 0]], 'context': [['Sherburne Ranger Station Historic District', ['The Sherburne Ranger Station in Glacier National Park is an example of the National Park Service Rustic style.', ' Located in the Swiftcurrent portion of the park, it was built in 1926.', ' It is part of a small historic district that includes a mess hall and subsidiary structures, formerly known as the Sherburne Road Camp, established in 1931.', ' The ranger station closely resembles the ranger stations at Belly River and Lake McDonald.', ' A checking station at the road remains substantially intact.']], ['Rising Sun Auto Camp', ['The Rising Sun Auto Camp, also known as the Roes Creek Auto Camp, East Glacier Auto Camp or simply Rising Sun preserves a portion of the built-up area of Glacier National Park that documents the second phase of tourist development in the park.', ' Rising Sun is located along the Going-to-the-Sun Road, approximately 7 mi from the east entrance to Glacier National Park, Montana, United States.', \" Rising Sun is a wayside area that has a National Park Service campground, a camp store and gift shop, picnic area, restaurant, as well as a motel and guest cabins which are managed by the park's concessionaire, Xanterra Parks & Resorts.\", ' In the immediate area, there is also a boat dock as well as sightseeing boats which allow visitors to tour Saint Mary Lake, the second largest lake in the park.', ' \"The most popular spot for [Glacier] tourists is Rising Sun, an overlook of Goose Island in St. Mary Lake and one of the most photographed spots in the park.\"']], ['Wanshan Archipelago Campaign', ['The Wanshan Archipelago Campaign (万山群岛战役) was a campaign fought between the communist and the nationalist forces during the Chinese Civil War for the control of Wanshan Archipelago (\"Wanshan Qundao\", 万山群岛), and resulted in communist victory.', ' The archipelago consists of 48 islands strategically located at the mouth of the Pearl River, a chokepoint on the communication lines to Hong Kong and Macau.', ' The largest island is the Laurel Mountain (Guishan, 桂山) Island, which was formerly known as Trash Tail (Lajiwei, 垃圾尾) Island.', \" Other major islands include Outer Linding (Wailinding, 外伶仃) Island, Dong'ao (东澳) Island, Tri-gate (Sanmen, 三门) Island, Greater Ten-thousand Mountain (Da Wanshan, 大万山) Island, Lesser Ten-thousand Mountain (Xiao Wanshan, 小万山) Island, Burden Pole (Dangan, 担杆) Islands, and Jianpeng (佳蓬) Islands.\"]], ['Birnie Island', ['Birnie Island is a small, uninhabited coral island, 20 hectares in area, part of the Phoenix Island group, that is part of the Republic of Kiribati.', ' It is located about 100\\xa0km SE of Kanton Island and 90\\xa0km WNW of Rawaki Island, formerly known as Phoenix Island.', ' It lies at .', ' Birnie island measures only 1.2\\xa0km long and 0.5\\xa0km wide.', ' There is no anchorage, but landing can be made on the lee beach.']], ['Marinelli Glacier', ['Marinelli Glacier is a tidewater glacier located in Alberto de Agostini National Park, Isla Grande de Tierra del Fuego.', ' The glacier spills out from the backbone of the Cordillera Darwin and calves into Ainsworth Bay, an embayment of the Almirantazgo Fjord.', ' The Marinelli Glacier is in a state of retreat, beginning at least as early as 1960 and continuing to the present time.']], ['Boaz Island, Bermuda', ['Boaz Island, formerly known as \"Gate\\'s Island\" or \"Yates Island\", is one of the six main islands of Bermuda.', ' It is part of a chain of islands in the west of the country that make up Sandys Parish, lying between the larger Ireland Island and Somerset Island, and is connected to both by bridges.', ' Its east coast forms part of the edge of the Great Sound.', ' Boaz Island was part of the Royal Naval base, which included the HM Dockyard on Ireland Island.', ' From 1939, Boaz Island was used as a Royal Naval Air Station.', ' Its primary role was the servicing, repair and replacement of spotter floatplanes and flying boats belonging to naval vessels.', \" Early in the Second World War, with no other units to fill the role, aeroplanes from Boaz Island were used to maintain anti-submarine air patrols, using whatever aircrew were on hand, including pilots from the Bermuda Flying School on Darrell's Island.\", ' All that remains of the Fleet Air Arm facility today is a hangar on runway road, and two slips.']], ['Sea Pines Resort', ['The Sea Pines Resort or Sea Pines is located in Sea Pines Plantation, a 5,200-acre private residential gated community located on the southern tip of the island which comprises the town of Hilton Head Island, South Carolina.', ' Sea Pines is home to four golf courses, including Harbour Town Golf Links, Atlantic Dunes by Davis Love III, (formerly known as the Ocean Course), the Heron Point golf course (formerly known as the Sea Marsh course) and the Sea Pines Country Club Course.', ' The RBC Heritage is a PGA Tour event held annually in April at the Harbour Town course.']], ['Isla Grande de Tierra del Fuego', ['Tierra del Fuego—literally \"Land of the Fire\", formerly \"Isla de Xativa\" and also known as Isla Grande de Tierra del Fuego—is an island near the southern tip of South America from which it is separated by the Strait of Magellan.', ' The western portion (61.43%) of the island (29,484.7 km2 ) is in Chile (Province of Tierra del Fuego and Antártica Chilena Province), while the eastern portion (38.57%, 18,507.3 km2 ) is in Argentina (Tierra del Fuego Province).', ' It forms the major landmass in an extended group of islands or archipelago also known as Tierra del Fuego.']], ['Douglas River', [\"The Douglas River, formerly known as the Twain, is a river of the West Coast of New Zealand's South Island.\", ' Its source is high in the Southern Alps, five kilometres south of Mount Sefton, and its upper reaches are fed by water from the Douglas Glacier.', ' It flows west for 18 kilometres, joined by runoff from the Horace Walker Glacier, before joining the waters of the Karangarua River.', \" The Douglas River's entire course is within Westland Tai Poutini National Park.\", ' The river and glacier are named after Charles Edward Douglas, a 19th-century explorer and mountaineer.']], ['Lian Island', ['Lian Island () is the largest island in Lianyungang, Jiangsu, China.', ' The island is located inside Haizhou Bay in the Yellow Sea.', ' It is 9 km long from east to west across the island and it has an area of 7.57 km2 .', ' 80% of the island is covered with forests.', ' The longest sea dyke nationally (6.7 km long) connects the island with the east of the city of Lianyungang.', ' Lian Island is the only AAAA-class seashore tourist attraction in Jiangsu.', ' The island was formerly known as Yingyou hill.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.619\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac11201554299294b21908c', 'answer': 'Oakland County', 'question': 'What county is Ron Teachworth from?', 'supporting_facts': [['Ron Teachworth', 0], ['Rochester Hills, Michigan', 0]], 'context': [['Ron Teachworth', ['Ronald S. Teachworth is an American artist, writer and film director from Rochester Hills, Michigan.']], ['Paulville', ['Paulville, Texas, is an American cooperative organization as well as the site and planned community under its development in the salt flats of north Hudspeth County, intended to consist exclusively of Ron Paul supporters.', ' The Paulville community idea was named after U.S. Congressman and 2012 presidential candidate Ron Paul, and the cooperative is modeled on his often libertarian ideas. \"', 'The New York Times\" says, \"For now, the town is little more than an idea and a title deed,\" but considers it to be evidence that Paul\\'s \"campaign appears to be growing into something beyond a conventional protest campaign,\" also echoing the concept expressed by others that \"the Ron Paul revolution has increasingly less to do with Ron Paul\".']], [\"Sequoyah County Sheriff's Office (Oklahoma)\", [\"Sequoyah County Sheriff's Office is the chief law enforcement agency that serves a population of over 42,391 people in Sequoyah County, Oklahoma.\", \" In 2007, the sheriff's department became the first in the state to arrest illegal immigrants under Oklahoma's new law, the Oklahoma Taxpayer and Citizen Protection Act of 2007.\", ' In 2009, the county jail was put back into the sheriffs department.', ' The jail was previously managed by the Sequoyah County Criminal Justice Authority.', \" Ron Lockhart's uncle, Sam Lockhart, is a former sheriff in the county.\", ' As of 2017, Larry Lane is the current Sheriff of Sequoyah County.']], ['Church of Scientology', ['The Church of Scientology is a multinational network and hierarchy of numerous ostensibly independent but interconnected corporate entities and other organizations devoted to the practice, administration and dissemination of Scientology, a new religious movement.', \" The Church of Scientology International (CSI) is officially the Church of Scientology's parent organization, and is responsible for guiding local Scientology churches.\", ' At a local level, every church is a separate corporate entity set up as a licensed franchise and has its own board of directors and executives.', ' The first Scientology church was incorporated in December 1953 in Camden, New Jersey by L. Ron Hubbard.', ' Its international headquarters are located at the Gold Base, in an unincorporated area of Riverside County, California.', ' The location at Gilman Hotsprings is private property and not accessible by the public.', ' Scientology Missions International is under CSI and oversees Scientology missions, which are local Scientology organizations smaller than churches.', ' The Church of Spiritual Technology (CST) is the organization which owns all the copyrights of the estate of L. Ron Hubbard.']], ['Ron Ehrenreich', ['Ron Ehrenreich (born 1950) is an American credit union officer and teacher.', ' He was the Vice-Presidential candidate for the Socialist Party USA in the United States presidential election, 1988, as the running mate of Willa Kenoyer.', ' The ticket received 3,882 votes, 2,587 of the votes came from New Jersey.', ' He has been the treasurer of the Syracuse Cooperative Federal Credit Union since its opening in 1982.', ' He later ran as a Green Party candidate for Onondaga County, New York Comptroller in 1999.', ' Ron is married to Sondra Roth, and has two children, Hanah and Sam.']], ['Ron Stephens (Illinois politician)', ['Ron Stephens (born 1948) is a former Republican member of the Illinois House of Representatives, representing the 102nd district from 1985 to 1991, and from 1993 until 2011 when he announced his retirement.', ' The district includes portions of Bond County, Madison County, Effingham County, Fayette County and St. Clair County.', ' He was the Assistant Republican Leader in the state House until his retirement.']], ['Rochester Hills, Michigan', ['Rochester Hills is a city in northeast Oakland County of the U.S. state of Michigan, in the northern outskirts of Metropolitan Detroit area.', ' As of the 2010 census, the city had a total population of 70,995.']], ['Jasper County Community Unit School District 1', [\"Jasper County Community Unit School District 1 is a unified school district based in Jasper County's county seat of Newton, Illinois; it is the only school district in the county and is, consequently, the main educational body in all of Jasper County, although it serves portions of Effingham County and Cumberland County as well.\", ' This school district is composed of six schools in total; four elementary schools, one junior high school, and one high school.', ' There is also a prekindergarten program run at the high school of the district should parents wish to enroll their children early.', ' Willow Hill Elementary School, which is located in the village of its namesake, serves only kindergarteners; its proximity to the county seat and central position in the county allows its students to dawn from all parts of the county and still have easy access to the elementary school they will attend.', ' The principal of Willow Hill is Dave Parker.', ' Grove Elementary School is located in Island Grove, Illinois, the highest point of elevation in the county.', ' Grove Elementary School educates students from kindergarten to grade six, and it runs a prekindergarten program as well.', ' Craig Carr is the principal of this school.', ' Ste. Marie Elementary School is located in the southern Jasper County village of Ste. Marie, and serves students in grades one through six.', ' The principal of Ste. Marie is David Parker, the principal of Willow Hill Elementary School.', \" Newton Elementary School is located in the county seat of Newton, and serves most of west Jasper County's first through sixth graders under principal Travis Wyatt.\", ' The latter three elementary schools feed into Jasper County Junior High School and are taught in the facility during seventh and eighth grade while being supervised by Newton Elementary School principal Travis Wyatt before graduating into Newton Community High School.', ' Students in grades nine through twelve spend their last leg of precollegiate education at this school; their principal is Ruth Kerner.', \" The district superintendent is Ron Alburtus, and the district's mascot is the eagle.\"]], ['Ohio House of Representatives, 78th District', ['The Ohio House of Representatives, 78th District, is located in Pickaway County, Ohio, Hocking County, Ohio, Morgan County, Ohio, Fairfield County, Ohio and small parts of Athens and Muskingum counties, with the major population centers being Circleville, Ohio and Logan, Ohio, as well as several villages.', ' District 78 is primary a rural district, but falls within the Columbus Metro area.', \" The district's boundaries were last redrawn in 2012 and the seat has been held by Republican Ron Hood since its redistricting.\"]], ['Going Back (film)', ['Going Back is a 1983 American independent drama film written and directed by Ron Teachworth and starring Bruce Campbell and Christopher Howe.', ' It was Campbell\\'s second feature film, produced shortly after \"The Evil Dead\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.620\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a79118755429974737f7962', 'answer': 'Foo Fighters', 'question': 'Which band formed first, Awolnation or Foo Fighters?', 'supporting_facts': [['Awolnation', 1], ['Foo Fighters', 0]], 'context': [['The Colour and the Shape', ['The Colour and the Shape is the second studio album by the American rock band Foo Fighters.', \" Produced by Gil Norton, it was released through Capitol Records and the group's own Roswell Records on May 20, 1997.\", ' The record is the debut of the Foo Fighters as a group, as the band\\'s previous record, \"Foo Fighters\" (1995), was primarily recorded by frontman Dave Grohl and friend Barrett Jones as a demo.', ' After the project ballooned and became an international success, the group convened for pre-production in the fall of 1996 and brought in producer Norton to establish a pop sensibility for the tracks.', ' The band strived to create a full-fledged rock record, although the music press predicted another grunge offshoot.']], ['Foo Fighters discography', ['The discography of Foo Fighters, an American rock band formed in 1994 by Dave Grohl, consists of nine studio albums, four extended plays (EPs), six video albums, and 36 singles.', ' The current Foo Fighters line-up consists of Grohl (vocals and guitar), Taylor Hawkins (drums), Rami Jaffee (keyboard), Nate Mendel (bass), Chris Shiflett (guitar), and Pat Smear (guitar).']], ['The Nation Blue', ['The Nation Blue is a rock band formed in Tasmania and based in Melbourne, Victoria, noted for their intense live performances and bleak subject matter.', ' They have toured nationally in Australia and internationally in Brazil, America and Japan and have supported Helmet and Foo Fighters.', ' In the third song of their first set in support of the Foo Fighters, bass player, Matt Weston, dislocated his knee, but saw out the remainder of the set while lying painfully on the stage floor.']], ['Foo Fighters: Back and Forth', ['Foo Fighters: Back and Forth is a 2011 rockumentary about the American rock band Foo Fighters, directed by filmmaker James Moll.', ' The film documents the band\\'s history, and gets its title from a track on the Foo Fighters\\' seventh studio album \"Wasting Light\".']], ['Foo Fighters', ['Foo Fighters is an American rock band, formed in Seattle, Washington in 1994.', ' It was founded by Nirvana drummer Dave Grohl as a one-man project following the dissolution of Nirvana after the death of Kurt Cobain.', ' The group got its name from the UFOs and various aerial phenomena that were reported by Allied aircraft pilots in World War II, which were known collectively as \"foo fighters\".']], ['Sunny Day Real Estate', ['Sunny Day Real Estate was an American emo band from Seattle, Washington.', ' They were one of the early emo bands and helped establish the genre.', ' In 1994, the band released their debut album \"Diary\" on Sub Pop Records to critical acclaim.', ' However, shortly after releasing their second album \"LP2\", the band broke up, with members Nate Mendel and William Goldsmith joining Foo Fighters and Jeremy Enigk embarking on a solo career.', ' In 1997, they regrouped long enough to record two more studio albums and a live album but ultimately disbanded once again in 2001.', ' The band reunited again in 2009.', ' Bassist Nate Mendel, who chose to remain with Foo Fighters during the previous reunion in 1997, took part in this reunion.', ' In a 2013 interview with MusicRadar, Mendel said Sunny Day Real Estate was inactive.', ' According to Mendel, the band attempted to record a full-length album after the end of their reunion tour, but the sessions \"just fell apart\".', ' In 2014 the band released one song from those sessions, \"Lipton Witch,\" on a split 7\" vinyl with Circa Survive on Record Store Day.']], ['The Fire Theft', ['The Fire Theft was an American rock band from Seattle, Washington.', ' They were formed in 2001 by vocalist/guitarist Jeremy Enigk, bassist Nate Mendel, and drummer William Goldsmith, all of whom were previously members of Sunny Day Real Estate.', ' Mendel also plays bass for Foo Fighters, and Goldsmith drummed for Foo Fighters between 1995 and 1997.', ' This lineup was identical to the original line up of Sunny Day Real Estate but with the exception of guitarist Dan Hoerner.', ' While the band went on a hiatus in 2004, there was never an announcement of an official breakup.']], ['Awolnation', ['Awolnation is an American alternative rock band, formed and fronted by Aaron Bruno, formerly of Under the Influence of Giants, Home Town Hero, and Insurgence.', ' The band is signed to Red Bull Records, and their first EP, \"Back from Earth\", was released on iTunes on May 18, 2010.', ' They released their first studio album, \"Megalithic Symphony\", on March 15, 2011; it featured their most notable hit, \"Sail\", which peaked at #17 on the \"Billboard\" Hot 100, #4 on the \"Billboard\" Rock Songs chart, and #5 on the \"Billboard\" Alternative Songs chart.', ' The song has been certified 6× platinum by the RIAA and has sold 5,500,000 copies in the United States.', ' As of February 29, 2016, the album has been certified platinum.']], ['List of Foo Fighters band members', ['Foo Fighters is an American alternative rock band formed by multi-instrumentalist and vocalist Dave Grohl in 1994.', ' After recording the album \"Foo Fighters\" alone, Grohl enlisted guitarist Pat Smear, bassist Nate Mendel and drummer William Goldsmith to join the band.', ' This lineup recorded the second Foo Fighters album, 1997\\'s \"The Colour and the Shape\", although most of Goldsmith\\'s drum recordings were removed and re-recorded by Grohl for the release, after he left the band due to creative tensions.', ' Goldsmith was replaced by Taylor Hawkins, and Franz Stahl replaced Smear after he also departed, although he only remained in the band for a short period and did not record an album with the group.', ' The group\\'s third album, \"There Is Nothing Left to Lose\", featured Hawkins on drums and was released in 1999.']], ['Songs from the Laundry Room', ['Songs from the Laundry Room is an EP by American rock band Foo Fighters, released exclusively for Record Store Day 2015.', ' This EP and the Foo Fighters album are the two Foo Fighters releases upon which Dave Grohl plays all the instruments.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.621\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae72fd95542991e8301cbb4', 'answer': '1903', 'question': 'The King who opened Newcastle Civic Center in 1968 was born in what year?', 'supporting_facts': [['Newcastle Civic Centre', 2], ['Olav V of Norway', 0]], 'context': [['Civic Center/Grand Park station', ['Civic Center/Grand Park, formerly Civic Center, is a heavy-rail subway station in the Los Angeles County Metro Rail system.', ' It is located on Hill Street between 1st and Temple Streets in the Civic Center area of Downtown Los Angeles.', ' The station is officially named Civic Center/Grand Park/Tom Bradley after former Los Angeles mayor Tom Bradley, who had a pivotal role in turning the subway into reality.']], ['Newcastle Civic Centre', ['Newcastle Civic Centre is a local government building located in the Haymarket area of Newcastle upon Tyne, England.', ' It is the main administrative and ceremonial centre for Newcastle City Council.', ' Designed by the city architect, George Kenyon, the building was completed in 1967 and was formally opened by HM King Olav V of Norway on 14 November 1968.', ' It is a Grade II* listed building.', ' The Newcastle Civic Centre is the joint eighth tallest building in the city.']], ['Newcastle Civic Theatre', ['The Newcastle Civic Theatre, also known as The Civic, is a heritage-listed building located on Hunter Street, Newcastle in the Hunter region, in New South Wales, Australia.', \" Opened in 1929 as a cinema, the 1520-seat venue is now the venue for a wide range of musicals, plays, concerts and dance events each year and is the city's oldest surviving theatre.\"]], ['XL Center', ['The XL Center (originally known as the Hartford Civic Center) is a multi-purpose arena and convention center located in downtown Hartford, Connecticut.', ' It is owned by the City of Hartford and operated by Spectra.', \" In December 2007, the Center was renamed when the arena's naming rights were sold to XL Group insurance company in a 6-year agreement.\", ' The arena is ranked the 28th largest among college basketball arenas.', ' Opened in 1974 as the Hartford Civic Center and originally located adjacent to Civic Center Mall, which was demolished in 2004.', ' It consists of two facilities: the Veterans Memorial Coliseum and the Exhibition Center.']], ['Colorado State Capitol', ['The Colorado State Capitol Building, located at 200 East Colfax Avenue in Denver, Colorado, United States, is the home of the Colorado General Assembly and the offices of the Governor of Colorado and Lieutenant Governor of Colorado.', ' The building is intentionally reminiscent of the United States Capitol.', ' Designed by Elijah E. Myers, it was constructed in the 1890s from Colorado white granite, and opened for use in November 1894.', ' The distinctive gold dome consists of real gold leaf, first added in 1908, commemorating the Colorado Gold Rush.', \" The building is part of Denver's Civic Center area.\", ' It was listed on the National Register of Historic Places as part of the Civic Center Historic District in 1974, and became part of the Denver Civic Center National Historic Landmark District in 2012.']], ['Mid-Hudson Civic Center', ['Mid-Hudson Civic Center is a venue located in Poughkeepsie, New York, consisting of Mair Hall (a concert and convention hall) and the McCann Ice Arena (an ice skating venue).', ' It was built in the 1970s as part of the general attempt at rehabilitation of the central district of the City of Poughkeepsie.', ' It is located at 14 Civic Center Plaza, on a segment of what was formerly known as Market Street near the former Main Mall.', ' The Poughkeepsie Grand Hotel, at 40 Civic Center Plaza, is adjacent on the same block and was originally designed to be constructed concurrently with the civic center and financed by Hilton, but the hotel construction was abandoned after the foundation was laid.', ' Four years after the completion of the Mid-Hudson Civic Center, Radisson Hotels bought the hotel property and after a re-design of the original hotel plans, construction of the hotel resumed.']], ['Charleston Civic Center', ['Charleston Civic Center is a municipal complex located in the downtown area of Charleston, West Virginia.', ' Originally completed in 1959 at the cost of $2.5 million, the Charleston Civic Center has undergone numerous renovations and expansions.', ' The Charleston Civic Center currently consists of three main components: the Civic Center Coliseum, the Little Theater, and the Charleston Convention Center, also referred to as the Grand Hall.']], ['Civic center', ['A civic center or civic centre is a prominent land area within a community that is constructed to be its focal point or center.', ' It usually contains one or more dominant public buildings, which may also include a government building.', ' Recently, the term \"civic center\" has been used in reference to an entire central business district of a community or a major shopping center in the middle of a community.', ' In this type of civic center, special attention is paid to the way public structures are grouped and landscaped.']], ['Wallace Civic Center', ['The George R. Wallace Jr.', ' Civic Center, more commonly known as the Wallace Civic Center, or just simply the Civic Center, is a 1,000-seat multi-purpose arena in Fitchburg, Massachusetts and has an end-stage concert capacity of 3,200.', ' It hosts various local concerts and sporting events for the area.', ' First opened in 1970, the Wallace Civic Center consists of the Gaetz Arena, the Landry Arena, a planetarium, and several multi-use banquet rooms.']], ['Olav V of Norway', ['Olav V (born Prince Alexander of Denmark; 2 July 1903 – 17 January 1991) was King of Norway from 1957 until his death.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.621\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8eea4a5542990e94052bb7', 'answer': 'between the 8th and 16th centuries', 'question': 'When was the Western Germanic language spoken from which the small settlement situated on the river Leda opposite Leer derives its name?', 'supporting_facts': [['Leda (river)', 3], ['Old Frisian', 0]], 'context': [['German language', ['German (\"Deutsch\" ] ) is a West Germanic language that is mainly spoken in Central Europe.', ' It is the most widely spoken and (co-) official language in Germany, Austria, Switzerland, South Tyrol (Italy), the German-speaking Community of Belgium, and Liechtenstein.', ' It is also one of the three official languages of Luxembourg.', ' The languages which are most similar to German are the other members of the West Germanic language branch: Afrikaans, Dutch, English, the Frisian languages, Low German/Low Saxon, Luxembourgish, and Yiddish.', ' German is the second most widely spoken Germanic language, after English.']], ['English language', ['English is a West Germanic language that was first spoken in early medieval England and is now a global \"lingua franca\".', ' Named after the Angles, one of the Germanic tribes that migrated to England, it ultimately derives its name from the Anglia (Angeln) peninsula in the Baltic Sea.', ' It is closely related to the Frisian languages, but its vocabulary has been significantly influenced by other Germanic languages, particularly Norse (a North Germanic language), as well as by Latin and Romance languages, particularly French.']], ['Proto-Norse language', ['Proto-Norse (also called Proto-Scandinavian, Primitive Norse, Proto-Nordic, Ancient Nordic, Ancient Scandinavian, Old Nordic, Old Scandinavian, Proto-North Germanic, North Proto-Germanic or Common Scandinavian) was an Indo-European language spoken in Scandinavia that is thought to have evolved as a northern dialect of Proto-Germanic in the first centuries CE.', ' It is the earliest stage of a characteristically North Germanic language, and the language attested in the oldest Scandinavian Elder Futhark inscriptions, spoken around from the 2nd to 8th centuries (corresponding to the late Roman Iron Age and the Germanic Iron Age).', ' It evolved into the dialects of Old Norse at the beginning of the Viking Age in about 800, which later themselves evolved into modern North Germanic languages.']], ['Dutch language', ['Dutch (\\xa0\\xa0 ) is a West Germanic language that is spoken by around 24 million people as a first language—including the population of the Netherlands and about sixty percent that of Belgium—and by another 5 million as a second language.', ' It is the third most widely spoken Germanic language, after English and German.']], ['Old Frisian', ['Old Frisian is a West Germanic language spoken between the 8th and 16th centuries in the area between the Rhine and Weser on the European North Sea coast.', \" The Frisian settlers on the coast of South Jutland (today's Northern Friesland) also spoke Old Frisian but no medieval texts of this area are known.\", ' The language of the earlier inhabitants of the region between the Zuiderzee and Ems River (the Frisians mentioned by Tacitus) is attested in only a few personal names and place-names.', ' Old Frisian evolved into Middle Frisian, spoken from the 16th to the 19th century.']], ['Afrikaans', ['Afrikaans ( ) is a West Germanic language spoken in South Africa, Namibia and, to a lesser extent, Botswana and Zimbabwe.', ' It evolved from the Dutch vernacular of South Holland (Hollandic dialect) spoken by the mainly Dutch settlers of what is now South Africa, where it gradually began to develop distinguishing characteristics in the course of the 18th century.', ' Hence, it is a daughter language of Dutch, and was previously referred to as \"Cape Dutch\" (a term also used to refer collectively to the early Cape settlers) or \"kitchen Dutch\" (a derogatory term used to refer to Afrikaans in its earlier days).', ' However, it is also variously described as a creole or as a partially creolised language.', ' The term is ultimately derived from Dutch \"Afrikaans-Hollands \" meaning \"African Dutch\".', ' It is the first language of most of the Afrikaners and Coloureds of Southern Africa.']], ['Leer', ['Leer is a town in the district of Leer, the northwestern part of Lower Saxony, Germany.', ' It is situated on the river Leda, a tributary of the river Ems, near the border with the Netherlands.']], ['Suorva', ['Suorva or Suorvadammen (the Suorva Dam) is a small settlement situated at the southern parts of Akkajaure, in Stora Sjöfallet National Park, Sweden.', ' The settlement can be reached by car (and bus, from Gällivare).', ' It consists of a few houses and a dam operated by Vattenfall, which regulates the flow to the hydroelectric plant in Vietas located about 5 kilometers downstream.', ' The road over the dam is normally open for hikers (not cars) and makes for a possible route into the northern parts of Sarek National Park which does not require using a boat.']], ['Old Saxon', ['Old Saxon, also known as Old Low German, was a Germanic language and the earliest recorded form of Low German (spoken nowadays in Northern Germany, the northeastern Netherlands, southern Denmark, the Americas and parts of Eastern Europe).', ' It is a West Germanic language, closely related to the Anglo-Frisian languages.', ' It has been documented from the 8th century until the 12th century, when it gradually evolved into Middle Low German.', ' It was spoken throughout modern northwestern Germany, primarily in the coastal regions and in the eastern Netherlands by Saxons, a Germanic tribe who inhabited the region of Saxony.', \" It partially shares Anglo-Frisian's (Old Frisian, Old English) Ingvaeonic nasal spirant law which sets it apart from Low Franconian and Irminonic languages, such as Dutch, Luxembourgish and German.\"]], ['Leda (river)', ['The Leda is a river in north-western Germany in the state of Lower Saxony.', ' It is a right tributary of the Ems and originates at the confluence of the Sagter Ems and the Soeste (Dreyschloot) near the town of Barßel.', ' The Leda flows into the Ems near the town of Leer.', ' On the southern bank of the Leda, in the \"Overledingen Land\" (Overledingen=\"country over the Leda\"), opposite Leer, lies the small settlement of Kloster Muhde (\"Muhde\" from the Old Frisian \"mutha\" meaning \"(river) mouth\").', ' The total length of the river is 29 km , of which the lower 1.9 km are navigable for sea-going vessels.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.622\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab430fa5542991751b4d6dd', 'answer': 'pan-Asian life insurance group', 'question': 'What kind of company is this group whose headquarters are in AIA Central? ', 'supporting_facts': [['AIA Central', 0], ['AIA Group', 0]], 'context': [['Freeport-McMoRan Center', ['Freeport-McMoRan Center (formerly One Central Park East) is a highrise located in Downtown Phoenix, Arizona.', \" It is located adjacent to Arizona State University's Downtown campus.\", ' Upon completion in 2009, the building was the first high-rise office tower to open in Downtown Phoenix in nearly eight years.', ' It is named for mining company Freeport-McMoRan whose headquarters are located in the building.']], ['Great Indian Peninsula Railway', ['The Great Indian Peninsula Railway was a predecessor of the Central Railway, whose headquarters was at the Boree Bunder in Mumbai (later, the Victoria Terminus and presently the Chhatrapati Shivaji Terminus).', ' The Great Indian Peninsula Railway was incorporated on August 1, 1849 by an act of the British Parliament.', ' It had a share capital of 50,000 pounds.', ' On August 17, 1849 it entered into a formal contract with the East India Company for the construction and operation of an railway line, 56\\xa0km long, to form part of a trunk line connecting Bombay with Khandesh and Berar and generally with the other presidencies of India.', ' The Court of Directors of the East India Company appointed James John Berkeley as Chief Resident Engineer and C. B. Kar and R. W. Graham as his assistants.', \" It was India's first passenger railway, the original 21 mile (33.8\\xa0km) section opening in 1853, between Bombay (Mumbai) and Tannah (Thane).\", ' On July 1, 1925 its management was taken over by the Government.', ' On November 5, 1951 it was incorporated into the Central Railway.']], ['Peter Parker House', ['The Peter Parker House, also known as the former headquarters of the Carnegie Endowment for International Peace, is a historic row house at 700 Jackson Place NW in Washington D.C. Built in 1860, it is historically significant for its association with the Carnegie Endowment, whose headquarters it was from its founding in 1910 until 1948.', ' The building was declared a National Historic Landmark in 1974.', ' It has since been incorporated into the Blair House complex serving high-profile official visitors to the capital.']], ['Furama Hong Kong Hotel', ['The Furama Hong Kong Hotel (), once known as the Furama Kempinski Hotel, was a 33-storey hotel in Central, Hong Kong, located at 1 Connaught Road Central.', ' The hotel was known for its revolving restaurant on the top floor.', ' The site is now occupied by the AIA Central office building.']], ['AIA Central', ['The AIA Central (), formerly called AIG Tower (), in Hong Kong is a 185-metre (607 ft.), 37-storey skyscraper that was completed in 2005 and serves as the headquarters of AIA Group.', ' It is located in Central, not far from the landmark Bank of China Tower.']], ['Continental Steel Corporation', ['The Continental Steel Corporation was United States steel producer from 1927 until 1986.', ' The company was created on June 21, 1927 through the merger of the Kokomo Steel and Wire Company (founded in Kokomo, Indiana in 1901) with the Superior Sheet Steel Company of Canton, Ohio, and the Chapman Price Steel Company of Indianapolis.', ' Among the original eleven directors was John E. Fredrick, who had been an organizer of the Kokomo Fence Machine Company in 1896 and had served on the board of directors of the Kokomo Steel and Wire Company.', ' Fredrick served as first Chairman of the Board of Continental Steel, whose headquarters were established in Kokomo.', ' The Kokomo operations of this corporation, however, continued to employ the Kokomo Steel and Wire name until 1944.', ' Continental Steel was dissolved in 1986, due to bankruptcy.']], ['Automated Imaging Association', [\"Automated Imaging Association (AIA) is the world's largest machine vision trade group.\", ' AIA has more than 330 members from 32 countries, including system integrators, camera, lighting and other vision components manufacturers, vision software providers, OEMs and distributors.', \" The association's headquarters is located in Ann Arbor, Michigan.\"]], ['Sugar refinery of Tienen', ['The Sugar refinery of Tienen (Dutch: Tiense Suikerraffinaderij - French: \"Raffinerie Tirlemontoise\"), a subsidiary of \"Raffinerie Tirlemontoise Group\" (RT Group), is a Belgian sugar producing company.', ' The company whose headquarters is located in Tienen (Belgium) has four business units: sugar activities, Orafti, Surafti and PPE, which together employ more than 1,800 people.']], ['Ace Trucking Co.', ['Ace Trucking Co. is a comedy science fiction series that featured in the comic \"2000 AD\" from 1981 to 1986.', ' Created by writers John Wagner and Alan Grant and artist Massimo Belardinelli, it followed the misadventures of a space trucking company headed by Ace Garp, a pointy-headed alien who spoke in a kind of futuristic CB radio slang.', ' The title was lifted from a 1970s improvisational comedy group whose membership had included Fred Willard, Patti Deutsch, Michael Mislove, George Memmoli, and Bill Saluga.']], ['AIA Group', ['known as AIA () is the largest independent public listed pan-Asian life insurance group.', ' It has a presence in 18 markets in Asia-Pacific, wholly owned branches and subsidiaries in Hong Kong, Thailand, Singapore, Malaysia, China, Korea, the Philippines, Australia, Indonesia, Taiwan, Vietnam, New Zealand, Macau, Brunei, a 97% subsidiary in Sri Lanka, a 49% joint venture in India, and representative offices in Myanmar and Cambodia.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.623\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7e08b55542997cc2c474fe', 'answer': 'Mario Lemieux', 'question': 'What current NHL team owner played for the team that won the Stanley cup in 1992?', 'supporting_facts': [['1992 Stanley Cup Finals', 2], ['Mario Lemieux', 0]], 'context': [['Gordon Haidy', ['Gordon Adam \"Gord\" Haidy (April 11, 1928 – October 6, 2004) was a professional ice hockey player who played right wing, shooting right.', ' He was born in Windsor, Ontario.', ' Gordon played only one game, in the semi-finals, for the Stanley Cup champion Detroit Red Wings in 1950.', \" Haidy's name was not included on the Stanley Cup, because he was not a regular member of the Red Wings.\", ' He did qualify to be on the cup, however, so the NHL credits him with winning one Stanley Cup.', ' Haidy never played another game in the NHL but played for several more years in the minors and at the senior hockey level for the Ontario Hockey League.', ' Haidy is one of only three players who played their only NHL game in the playoffs for a Stanley Cup winning team (See Doug McKay, Chris Hayes).', ' Haidy and his teammate Doug McKay accomplished the same feat of the winning both the Calder Cup and Stanley Cup in the same season.']], ['List of Montreal Canadiens seasons', ['The Montreal Canadiens (French: \"Les Canadiens de Montréal\" ) are a professional ice hockey team based in Montreal, Quebec.', ' They are members of the Atlantic Division in the Eastern Conference of the National Hockey League (NHL) and are one of the Original Six teams of the league.', ' The club is officially known as \"le Club de hockey Canadien \".', \" Founded in 1909, they have played a total of 107 seasons, 8 with the National Hockey Association (NHA) and 99 with the NHA's successor, the NHL.\", ' They are the only club to have played every season for both leagues and the only active NHL team to pre-date the founding of that league.', \" They have won the Stanley Cup 24 times, once under the NHA and 23 times since the founding of the NHL, and have also won 11 O'Brien Cup titles, 24 division championships, and 8 conference championships.\", ' Overall they have the most games played, most wins, most ties, most points, highest points percentage, most years in the playoffs, most division championships, and most Stanley Cup championships of any team in the NHL.']], ['1931 Stanley Cup Finals', ['The 1931 Stanley Cup Finals was played between the Montreal Canadiens and the Chicago Black Hawks, making their first Stanley Cup Finals appearance.', ' The Canadiens, defending champions, won the series to become the second NHL team to repeat as champion.', \" Former player and now coach, Chicago's Dick Irvin, made his Finals coaching debut against the team he would later coach to three Stanley Cup titles.\"]], ['1998 Stanley Cup Finals', [\"The 1998 Stanley Cup Finals was the championship series of the National Hockey League's (NHL) 1997–1998 season, and the culmination of the 1998 Stanley Cup playoffs.\", ' It was contested by the Western Conference champion and defending Stanley Cup champion Detroit Red Wings and the Eastern Conference champion Washington Capitals.', ' It was the 105th year of the Stanley Cup being contested.', \" The series was the Capitals' first appearance in a Stanley Cup Final since the franchise's inception in 1974.\", ' The Red Wings won the series for the second year in a row, four games to none.', \" It was the Wings' ninth Stanley Cup, and the most recent time when a Finals concluded with a sweep (as of 2017).\", ' This was also the last time until that a Stanley Cup Finals ended after an NBA Finals in the same season had concluded.', ' Detroit coach Scotty Bowman won his eighth Stanley Cup in that capacity (having previously done so with the Montreal Canadiens in , , , , and , the Pittsburgh Penguins in , and the Wings the previous year), tying him with former Canadiens coach Toe Blake for the record of most Cups won by a coach (which he would break when he helped the Red Wings win the 2002 Cup).']], ['Mario Lemieux', ['Mario Lemieux, OC, CQ ( ; ] ; born October 5, 1965) is a Canadian former professional ice hockey player and current owner of the Pittsburgh Penguins team.', \" He played parts of 17 seasons in the National Hockey League's (NHL) with the Pittsburgh Penguins, between 1984 and 2006.\", ' Dubbed \"The Magnificent One\" or \"Le Magnifique\" (as well as \"Super Mario\"), he is widely acknowledged to have been one of the best players of all time.', ' He is the only player to score one goal in each of the five possible situations in a single NHL game, a feat he accomplished in 1988.', ' A gifted playmaker and fast skater despite his large size, Lemieux often beat defencemen with fakes and dekes.']], ['1917–18 Toronto Hockey Club season', ['The 1917–18 Toronto Hockey Club season was the first season of the new \"Toronto\" franchise in the newly organized National Hockey League (NHL).', ' The team was intended as a \\'temporary\\' franchise, operating without an official club nickname (the press would dub them the \"Blue Shirts\" or \"Torontos\", and in 1948 the NHL would engrave \"Toronto Arenas\" on the Stanley Cup as the 1917–18 winner) and without a formal organization separate from the Toronto Arena Company that managed the Arena Gardens.', ' Despite this, the team came together to win the first NHL Championship, competing against existing teams that had transferred directly from the National Hockey Association (NHA).', \" Toronto would go on to win the Stanley Cup by defeating the Pacific Coast Hockey Association champion Vancouver Millionaires – the first Stanley Cup for an NHL team and the second Cup for a Toronto team after the Toronto Blueshirts' victory in the 1913–14 season of the NHA.\"]], ['1984 Stanley Cup Finals', ['The 1984 Stanley Cup Final was held between the Edmonton Oilers and the then-defending champion New York Islanders.', ' The Islanders had swept the Oilers in four straight games to win the Cup.', \" In 1984, the Islanders were seeking their fifth consecutive Stanley Cup championship, but the upstart Oilers would win the best-of-seven series four games to one to win their first Stanley Cup, becoming the third post-1967 expansion team and first former World Hockey Association team to win the Cup, and also the first team based west of Chicago to win the Cup since the WCHL's Victoria Cougars became the last non-NHL team to win it in .\", ' It was also the fifth straight Finals of teams that joined the NHL in 1967 or later and a rematch of the 1983 Finals—a Stanley Cup Finals rematch would not happen again until the Finals.', \" s of 2017 , the Islanders' four consecutive Cup wins (, , , 1983) and their appearance in the 1984 Cup Finals is an NHL record of 19 consecutive playoff series wins that currently stands unbroken.\", ' This would be the second of eight consecutive Finals contested by a team from Alberta (the Oilers appeared in six, the Calgary Flames in two), and the first of five consecutive Finals to end with the Cup presentation on Alberta ice (the Oilers won four times, the Montreal Canadiens one).']], ['1992 Stanley Cup Finals', ['The 1992 Stanley Cup Finals NHL championship series was contested by the Prince of Wales Conference and defending Stanley Cup champion Pittsburgh Penguins and the Clarence Campbell Conference champion Chicago Blackhawks.', ' The Blackhawks were appearing in their first Finals since .', ' After the Blackhawks jumped to an early 4–1 lead in the first game of the series, Mario Lemieux and the Penguins came back to win the game, sweep the series in four games, and win their second consecutive and second overall Stanley Cup.', ' It was the 99th year of the Stanley Cup, and the first to extend into the month of June.', ' It was the last final for Chicago Stadium as it closed in 1994.']], ['1983 Stanley Cup Finals', ['The 1983 Stanley Cup Finals was contested by the Edmonton Oilers in their first-ever Finals appearance and the defending champion New York Islanders, in their fourth, and fourth consecutive, Finals appearance.', ' The Islanders would win the best-of-seven series four games to none, to win their fourth-straight and fourth-overall Stanley Cup.', ' It was also the fourth straight Finals of post-1967 expansion teams, and the first involving a former World Hockey Association (WHA) team.', ' This is also the most recent time that a defending Stanley Cup champion has won the cup four years in a row, and also the first (and, to date, only) time a North American professional sports team has won four consecutive titles in any league competition with more than twenty teams.', ' Since 1983, no professional sports team on the continent has managed to win four straight championships and no NHL team has won more than two consecutive championships (most recently the Pittsburgh Penguins in and ).']], ['History of the National Hockey League', ['The history of the National Hockey League begins with the end of its predecessor league, the National Hockey Association (NHA), in 1917.', ' After unsuccessfully attempting to resolve disputes with Eddie Livingstone, owner of the Toronto Blueshirts, executives of the three other NHA franchises suspended the NHA, and formed the National Hockey League (NHL), replacing the Livingstone team with a temporary team in Toronto, the Arenas.', \" The NHL's first quarter-century saw the league compete against two rival major leagues—the Pacific Coast Hockey Association and Western Canada Hockey League—for players and the Stanley Cup.\", ' The NHL first expanded into the United States in 1924 with the founding of the Boston Bruins, and by 1926 consisted of ten teams in Ontario, Quebec, the Great Lakes region, and the Northeastern United States.', ' At the same time, the NHL emerged as the only major league and the sole competitor for the Stanley Cup; in 1947, the NHL completed a deal with the Stanley Cup trustees to gain full control of the Cup.', \" The NHL's footprint spread across Canada as Foster Hewitt's radio broadcasts were heard coast-to-coast starting in 1933.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.624\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adeaec455429939a52fe952', 'answer': 'August 28, 1774', 'question': 'The saint in \"A Time for Miracles\" was born on what date?', 'supporting_facts': [['A Time for Miracles', 0], ['Elizabeth Ann Seton', 0]], 'context': [['John Thayer (priest)', ['The Reverend John Thayer (1755–5 February 1815) was the first native of New England ordained to the Roman Catholic priesthood.', ' He was born in Boston, Massachusetts.', ' Thayer was educated at Yale College and was a Protestant in his early life.', ' He was ordained as a Congregationalist minister and served as a chaplain during the American Revolutionary War.', ' While visiting Rome in 1783, he converted to the Roman Catholic faith, an act which caused a sensation in New England at the time.', ' He credited his conversion to miracles attributed to the noted mendicant, Saint Benedict Joseph Labre, who lived and died there in that period.']], ['Fortunatus of Todi', ['Saint Fortunatus (died 537) was a 6th-century bishop of Todi.', ' According to tradition, he defended Todi during a Gothic siege.', ' He is the patron saint of Todi.', ' He is praised by Gregory the Great, who calls him a man of great virtue who took great care in attending to the sick.', \" Gregory, who was born around the time that Fortunatus died, was greatly interested in Fortunatus' life.\", ' Gregory writes that \"a certain poor old man was brought to me –because I always love to talk with such men- of whom I inquired his country, and hearing that he was of the city of Todi, I asked him whether he knew Bishop Fortunatus.', ' He said he knew him very well.', ' \\'Then I beseech you,\\' said I, \\'tell me whether you know of any miracles that he did, and, since I am very desirous to know, explained to me what manner of man he was.\\'\"']], ['A Time for Miracles', [\"A Time For Miracles is a 1980 American made-for-television biographical drama film chronicling the life story of America's first native born saint, Elizabeth Ann Bayley Seton.\", ' It was produced by ABC Circle Films for the American Broadcasting Company and telecast December 21, 1980, as a Christmas special.', \" The film was created by Beverlee Dean and directed by Michael O'Herlihy.\", ' The script was written by Henry Denker with collaboration with Sister Mary Hilaire and filmed in Georgia.', ' \"A Time For Miracles\" starred \"Ryan\\'s Hope\" and \"\" actress Kate Mulgrew as Elizabeth Seton.', ' John Forsythe and Lorne Greene also star.']], ['Palladius of Embrun', ['Saint Palladius of Embrun (French: \"Pallade, Pélade\" , Catalan: \"Patllari, Pal·ladi\" ) (d. ca. 541 AD) was a 6th-century bishop of Embrun.', ' Born to a Christian family, he studied under Catulin, bishop of Embrun, who had attended the Council of Épaone in 517.', ' When the Arians and Sigismund of Burgundy opposed the council, Catulin was exiled to Vienne.', ' Palladius accompanied him there, and took the opportunity to extensively study Scripture.', ' Palladius was ordained a priest and, according to legend, gained the gift of prophecy.', ' He is said to have predicted the fall and death of Sigismund.', ' Catulin died around 518, and Palladius would later be elected bishop of Embrun.', ' During his episcopate he built numerous churches, in Chorges, Sauze, and Rama, as well as sanctuaries dedicated to Saint Martin of Tours, and Saints Vincent, Orontius, and Victor, as well as to Genesius of Arles.', ' One source states that Palladius \"possessed an exceptional efficacy in obtaining whatever he petitioned God for.\"', ' Many miracles were attributed to him, and, besides the gift of prophecy, he enjoyed a \"mystical familiarity with the angels... [and] successfully defeated the machinations of the devil simply by making the sign of the cross.\"']], ['Elizabeth Ann Seton', ['Elizabeth Ann Bayley Seton, S.C., (August 28, 1774 – January 4, 1821) was the first native-born citizen of the United States to be canonized by the Roman Catholic Church (September 14, 1975).', \" She established the first Catholic girls' school in the nation in Emmitsburg, Maryland, where she also founded the first American congregation of religious sisters, the Sisters of Charity.\"]], ['Saint Menas', ['Saint Menas (also Minas, Menas, Mena, Mennas) (285 – c. 309), the Martyr and Wonder-worker, is one of the most well-known Egyptian saints in the East and the West, due to the many miracles that are attributed to his intercession and prayers.', ' Menas was an Egyptian soldier in the Roman army martyred because he refused to recant his Christian faith.', ' The common date of his commemoration is November 11, which occurs 13 days later (November 24) on the Julian calendar.']], ['Gregory of Tours', ['Saint Gregory of Tours (30 November c. 538 – 17 November 594) was a Gallo-Roman historian and Bishop of Tours, which made him a leading prelate of Gaul.', ' He was born Georgius Florentius and later added the name Gregorius in honour of his maternal great-grandfather.', ' He is the primary contemporary source for Merovingian history.', ' His most notable work was his \"Decem Libri Historiarum\" (\"Ten Books of Histories\"), better known as the \"Historia Francorum\" (\"History of the Franks\"), a title that later chroniclers gave to it, but he is also known for his accounts of the miracles of saints, especially four books of the miracles of St. Martin of Tours.', \" St. Martin's tomb was a major pilgrimage destination in the 6th century, and St. Gregory's writings had the practical effect of promoting this highly organized devotion.\"]], ['Lytham Priory', ['Lytham, a settlement recorded in the \"Domesday Book\" of 1086 as \"Lidun\", is situated on the Fylde coast, at the mouth of the River Ribble.', ' By the 12th century, there was already a church at Lytham, dedicated to Saint Cuthbert.', ' Between 1189 and 1194, Richard Fitz Roger of Woodplumpton gave the church and his land at Lytham to the monks of Durham Priory for the foundation of a Benedictine cell.', ' Richard already had a personal connection to Durham; he was said to have experienced two miracles ascribed to Saint Cuthbert (whose cult was centred at Durham) and had previously travelled there to give thanks for those miracles.', ' This may have been his motivation for donating land to that priory.', ' It is unlikely that the monks of Durham held any land at Lytham up to that point.', ' As a house dependent on Durham, Lytham Priory was small, with only two or three monks at a time.']], ['Lutgardis', ['Saint Lutgardis of Aywières (Dutch: \"Sint-Ludgardis\" ; 1182 – 16 June 1246; also spelled Lutgarde) is a saint from the medieval Low Countries.', ' She was born in Tongeren, known as \"Tongres\" in French (which is why she is also called \"Lutgardis of Tongres\" or \"Luitgard of Tonger(e)n\"), and entered into religious orders at the age of twelve.', ' During her life various miracles were attributed to her, and she is known to have experienced religious ecstasies.', ' Her feast day is June 16.']], ['Wandelbert', ['Wandelbert (813 - d. after 850) was a Benedictine monk and theological writer.', ' Little is known of his personal history.', ' He was apparently a native of Francia, and in 839 he was already a monk at the Abbey of Prüm, where he died.', ' About this date Abbot Markward commissioned him to rewrite the old \"Life of St. Goar\" and to supplement it by an account of the miracles worked by the saint.', ' The life Wandelbert wrote is not without historical value.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.625\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a820897554299676cceb1f4', 'answer': 'Mike\" Mills', 'question': 'Which founding member drew critical acclaim as a bass guitarist with melodic basslines  on Murmur?', 'supporting_facts': [['Murmur (album)', 1], ['Mike Mills', 0]], 'context': [['Construct (album)', ['Construct is the tenth full-length studio album by Swedish melodic death metal band Dark Tranquillity.', ' It was released on 27 May 2013 through Century Media Records.', ' A music video for \"Uniformity\", directed by Patric Ullaeus, was released on 10 May 2013.', ' The album was written during what the band described as their \"darkest period\" and drew critical praise for its melodrama and darkness.', \" Construct would be the band's last studio album with founding member guitarist Martin Henriksson as he left the band in early 2016 due to loss of passion for playing music.\"]], ['Hope Clarke', ['Hope Clarke (born March 23, 1941) is an American actress, dancer, vocalist, choreographer, and director.', ' Clarke performed as principal dancer with the Katherine Dunham Company and the Alvin Ailey American Dance Theater, 1960s; actress on stage, film, and television, 1970s–1980s; choreographer and director, 1980s--.', ' Clarke served on the Tony Awards Nominating Committee for the 2011–12 Broadway season.', ' Clarke made history in 1995 when she became the first African American, as well as the first African-American woman, to direct and choreograph a major staging of the opera-musical \"Porgy and Bess\".', \" Clarke's production of the George Gershwin classic was staged in celebration of the work's 60h anniversary, and it toured not only major American cities but Japan and Europe as well.\", ' Clarke drew critical acclaim for her commitment to staging the show as a monument to African-American community and pride, giving a more hopeful, positive aura to a story that has been criticized for its stereotypes.', ' As for the director herself, the success of \"Porgy and Bess\" is just the latest accolade in a long career devoted to dance and drama.']], ['The Deep Dark Woods', ['The Deep Dark Woods are a Canadian alternative country band from Saskatoon, currently signed to Sugar Hill Records in the United States and Six Shooter Records in Canada.', ' First established in 2005, the band consisted of singer and guitarist Ryan Boldt, bass guitarist Chris Mason, guitarist Burke Barlow and drummer Lucas Goetz.', ' Pianist and organist Geoff Hilhorst joined the group in 2009 after the release of their breakout album \"Winter Hours\".', ' Founding member Burke Barlow played guitar until 2012 when he left the group and was replaced by Clayton Linthicum.', ' Founding Member Lucas Goetz left the group in late 2014, after which the band went on hiatus.', ' In 2017 Chris Mason officially left the group.']], ['Murmur (album)', ['Murmur is the debut studio album by the American alternative rock band R.E.M., released on April 12, 1983 by I.R.S. Records.', ' \"Murmur\" drew critical acclaim upon its release for its unusual sound, defined by lead singer Michael Stipe\\'s cryptic lyrics, guitarist Peter Buck\\'s jangly guitar style, and bass guitarist Mike Mills\\' melodic basslines.']], ['Mike Mills', ['Michael Edward \"Mike\" Mills (born December 17, 1958) is an American multi-instrumentalist, singer, and composer who was a founding member of the alternative rock band R.E.M. Though known primarily as a bass guitarist, backing vocalist, and pianist, his musical repertoire also includes keyboards, guitar, and percussion instruments.', \" He contributed to a majority of the band's musical compositions.\"]], ['Mathieu Amalric', ['Mathieu Amalric (] ; born 25 October 1965) is a French actor and filmmaker.', ' Amalric is perhaps best known internationally for his performance as the lead villain in Bond film \"Quantum of Solace\", his performance in Steven Spielberg\\'s \"Munich\", and for his role in \"The Diving Bell and the Butterfly\", for which he drew critical acclaim.', ' He has also won the César Award and the Lumières Award.']], ['Roster McCabe', ['Roster McCabe was an American rock band noted for its tight, high energy dance shows, exploration of music across genres and devoted fan base.', ' Named by Billboard Magazine as one of \"five up-and-coming jam bands that could draw audiences to the festivals of tomorrow\", Roster has toured nationally and played over 550 gigs in the last four years.', ' The band\\'s music blends elements of a wide variety of genres, including reggae, rock, funk, and jazz, although the band describes its sound as \"Funky Reggae Dance Rock\".', ' After the departure of founding member Drew Preiner, the band renamed themselves \"Night Phoenix.\"', ' Night Phoenix (Steele, Mullenburg, Peterson, and Daum) played a handful of shows in the winter of 2013–2014, before breaking up in early-mid 2014.', ' Their final show was performed on May 10, 2014, and was held at The Popcorn in La Crosse, Wisconsin.']], ['R.E.M.', ['R.E.M. was an American rock band that formed in Athens, Georgia, in 1980 by lead singer Michael Stipe, guitarist Peter Buck, bassist/backing vocalist Mike Mills, and drummer Bill Berry.', \" One of the first alternative rock bands, R.E.M. was noted for Stipe's particular vocal quality and obscure lyrics, Buck's ringing, arpeggiated guitar style, and Mills' melodic basslines and backing vocals.\", ' R.E.M. released its first single—\"Radio Free Europe\"—in 1981 on the independent record label Hib-Tone.', ' The single was followed by the \"Chronic Town\" EP in 1982, the band\\'s first release on I.R.S. Records.', ' In 1983, the group released its critically acclaimed debut album, \"Murmur\", and built its reputation over the next few years through subsequent releases, constant touring, and the support of college radio.', ' Following years of underground success, R.E.M. achieved a mainstream hit in 1987 with the single \"The One I Love\".', ' The group signed to Warner Bros.', ' Records in 1988, and began to espouse political and environmental concerns while playing large arenas worldwide.']], ['Stardust (video game)', [\"Stardust is a shoot 'em up computer game for the Amiga, released by the Finnish company Bloodhouse in 1993.\", ' The game is essentially an \"Asteroids\" clone with enhancements, such as power-ups, shields, a high-energy techno module soundtrack, vivid use of colors and the occasional tunnel section that revolves around a sphere.', \" The game's graphics drew critical acclaim for the aforementioned tunnels and the liberal use of ray-tracing.\", ' The company has since merged with Terramarque to form Housemarque.']], ['Peter Hofmann', ['Peter Hofmann (22 August 1944 – 30 November 2010) was a German tenor who had a successful performance career within the fields of opera, rock, pop, and musical theatre.', ' He first rose to prominence in 1976 as a heldentenor at the Bayreuth festival\\'s \"Jahrhundertring\" (\"Centenary Ring\") in 1976, where he drew critical acclaim for his performance of Siegmund in Richard Wagner\\'s \"Die Walküre\".', \" He was active as one of the world's leading Wagnerian tenors over the next decade, performing roles like Lohengrin, Parsifal, Siegfried, and Tristan at major opera houses and festivals internationally.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.625\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abfa9745542997ec76fd426', 'answer': '\"The Big Bang Theory\"', 'question': 'Young Sheldon, the tv series that follows the character Sheldon Cooper at the age of 9, is a spin-off from what prequel?', 'supporting_facts': [['Young Sheldon', 0], ['Young Sheldon', 1], ['Sheldon Cooper', 0]], 'context': [['Macross Ace Frontier', ['Macross Ace Frontier (マクロス エース フロンティア , makurosu ēsu furontia ) is a shooting game developed by Artdink for the PlayStation Portable.', ' The game is based on Studio Nue\\'s popular \"Macross\" series, one of which is known in the West as the first generation of \"Robotech\".', ' It features original elements as well as characters, mechanics, episode plots and BGM borrowed from 1982\\'s \"The Super Dimension Fortress Macross\" original TV series, the 1992 \"Macross Plus\" OVA series, the 1994 \"Macross 7\" TV series and the 2008 \"Macross Frontier\" TV series, as well as incorporating more elements from the 1984 \"\" feature film and the 2002 prequel OVA series \"Macross Zero\".']], ['Leonard Hofstadter', ['Leonard Leakey Hofstadter, Ph.D., is a fictional character in the CBS television series \"The Big Bang Theory\", in which he is portrayed by actor Johnny Galecki.', ' Leonard is an experimental physicist, who shares an apartment with colleague and best friend Dr. Sheldon Cooper (Jim Parsons).', ' For his portrayal, Galecki was nominated for a Primetime Emmy Award and a Golden Globe Award in 2011.']], ['Steven Molaro', ['Steven Molaro, also known as Steve Molaro, is an American television producer and writer.', ' He has worked on such productions as \"Freddie\", \"The Class\", \"Complete Savages\" and the Dan Schneider-produced series \"All That\", \"The Amanda Show\", \"What I Like About You\", \"Drake & Josh\", \"Zoey 101\" and \"iCarly\".', ' Since 2007, he has been a producer/writer on the sitcom \"The Big Bang Theory\".', ', Molaro also co-created it\\'s prequel spinoff, \"Young Sheldon\" with Chuck Lorre.']], ['Sheldon Cooper', ['Sheldon Lee Cooper, Ph.D., Sc.D.', ', is a fictional character in the CBS television series \"The Big Bang Theory\" and \"Young Sheldon\", portrayed by actor Jim Parsons in \"The Big Bang Theory\" and Iain Armitage in \"Young Sheldon\".', \" For his portrayal, Parsons has won four Primetime Emmy Awards, a Golden Globe Award, a TCA Award, and two Critics' Choice Television Awards.\", ' The childhood of the character is the focus of \"Young Sheldon\", the show being set in 1989, when 9-year-old Sheldon, who has skipped ahead four grades, starts high school alongside his older brother.']], ['Margo Harshman', ['Margo Cathleen Harshman (born March 4, 1986) is an American actress known for her role as Tawny Dean on the Disney Channel series \"Even Stevens\".', ' She is also known for her role on \"The Big Bang Theory\" as Sheldon Cooper\\'s assistant, Alex Jensen, and as Delilah Fielding on \"NCIS\".']], ['Jim Parsons', ['James Joseph Parsons (born March 24, 1973) is an American actor.', ' He is known for playing Sheldon Cooper in the CBS sitcom \"The Big Bang Theory\".', ' He has received several awards for his performance, including four Primetime Emmy Awards for Outstanding Lead Actor in a Comedy Series and the Golden Globe Award for Best Actor in a Television Series Musical or Comedy.']], ['Young Sheldon', ['Young Sheldon (stylized as young Sheldon) is an American television sitcom on CBS created by Chuck Lorre and Steven Molaro.', ' The series is a spin-off prequel to \"The Big Bang Theory\" and follows the character Sheldon Cooper at the age of 9, living with his family in East Texas and going to high school.', ' Iain Armitage stars as young Sheldon, alongside Zoe Perry, Lance Barber, Montana Jordan, and Raegan Revord.', ' Jim Parsons, who portrays an adult Sheldon Cooper on \"The Big Bang Theory\", narrates the series and serves as an executive producer.']], ['Euglossa bazinga', ['Euglossa bazinga is a euglossine bee species found in Brazil.', ' It is named after the catchphrase of the fictional character Dr. Sheldon Cooper from the television show \"The Big Bang Theory\".', ' It was previously misidentified as \"Euglossa ignita\", and is threatened with habitat loss.']], ['Damon Dark', ['Damon Dark is an independent public access TV series and web series from Australia about a heroic and obsessive investigator of UFO incidents and other strange cases, created by Australian writer, actor and film maker Adrian Sherlock.', ' Damon James Dark became a dedicated alien investigator after a close encounter during his teenage years.', ' He is associated with both the secret service and friendly aliens, including a character called Vincent Kosmos (an alien time traveller) and Trans-Dimensional Control (an alien law enforcement agency).', ' The character of Damon Dark has appeared in a 5-week TV series on Community TV 31 in Melbourne Australia, a self-published novel (\"Biodome\") on Amazon\\'s createspace platform and a long running web series on YouTube.', ' He has also been involved in related web series \"Young Damon Dark\" and \"Vincent Kosmos.\"', ' He has also been the focus of a one actor stage drama.', ' The character of Damon Dark has been played by Adrian Sherlock, Bruce Hughes, Aiden Sherlock and Jack Knoll.', ' Damon Dark is a loner, dresses in black, has a huge experience of aliens and their technology.', ' Damon is characterized by his high intelligence, idealistic moral outlook and wry sense of humor.', ' His best friend in the series is the long-suffering Gary Sutton, played by actor Robert Trott.', ' Damon Dark began in 1999, with a five-part weekly series on Melbourne\\'s Community TV 31, (although the pilot was shot in 1996 and the show had been in development since 1990) following a screening of a 65-minute version of the story \"Maddox\" at the 57th World Science Fiction Convention (Aussiecon Three) held in Melbourne.', ' The series was later revived as a YouTube webseries which inspired several related webseries, including \"The Young Damon Dark Adventures\" in which the character is played as a teenager, and Vincent Kosmos, (created by and starring Chris Heaven, , an Italian actor and musician, about a renegade alien character who is a friend of Damon.']], ['Penny (The Big Bang Theory)', ['Penny is a fictional character on the American CBS sitcom \"The Big Bang Theory\", portrayed by actress Kaley Cuoco.', ' She is the primary female character in the series, befriending her across-the-hall neighbors Leonard Hofstadter (Johnny Galecki) and Sheldon Cooper (Jim Parsons), two physicists who work at the nearby California Institute of Technology (Caltech).', \" Penny's lack of advanced education, but outgoing personality and common sense drastically contrast with the personalities of the primary male characters in the series, even though she is considered part of their group.\", ' She is the love interest of Leonard, with whom she maintains a brief romantic relationship during the third season, which is later resumed in the fifth season and culminates in an engagement at the end of the seventh season and a wedding at the start of season 9.', ' Penny is the only main character of the show whose last name has not been revealed, although she has been occasionally referred to or addressed with the last name Hofstadter since her wedding.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.626\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ade61fd554299728e26c704', 'answer': 'Disco', 'question': 'Get Off is an album by a group using which style originating in NYC in the early 1970s?', 'supporting_facts': [['Get Off (Foxy album)', 0], ['Disco', 0]], 'context': [['Disco', [\"Disco is a musical style originating in New York City in the early 1970s, and remained urban and largely underground until the middle of the decade when it began to emerge from America's urban nightlife scene, where it had been curtailed to house parties and makeshift discotheques, and began making regular appearances mainstream, gaining popularity and increasing airplay on radio.\", ' It achieved popularity during the mid-1970s to the early 1980s.', ' Its initial audiences in the U.S. were club-goers from the gay, African American, Italian American, Latino, and psychedelic communities in Philadelphia and New York City during the late 1960s and early 1970s.', ' Disco can be seen as a reaction against both the domination of rock music and the stigmatization of dance music by the counterculture during this period.', ' Disco was popular with both men and women from many different backgrounds, with dances including The Bump (1974), The Hustle (1975).']], ['New school (tattoo)', ['New school is a tattooing style originating as early as the 1970s and influenced by some features of old school tattooing in the United States.', ' The style is often characterized by the use of heavy outlines, vivid colors, and exaggerated depictions of the subject.', ' New school also represents a transition towards openness in the sharing of techniques in tattooing.']], ['Robert Goodnough', ['Robert Goodnough (October 23, 1917 – October 2, 2010) was an American abstract expressionist painter.', ' A veteran of World War II, Goodnough was one of the last of the original generation of the New York School; (although he has been referred to as a member of the \"second generation\" of Abstract Expressionists), even though he began exhibiting his work in galleries in New York City in the early 1950s.', ' Robert Goodnough was among the 24 artists from the total of 256 participants who were included in the famous 9th Street Art Exhibition, (1951) and in all the following New York Painting and Sculpture Annuals from 1953 to 1957.', ' These Annuals were important because the participants were chosen by the artists themselves.', ' Early in his career starting in 1950 he showed his paintings at the Wittenborn Gallery, NYC.', ' He had shown at the Tibor de Nagy Gallery in New York City from 1952 to 1970 and again from 1984 to 1986.', ' In 1960 and 1961 he had solo exhibitions at The Art Institute of Chicago.', ' A veteran of scores of solo exhibitions and hundreds of group exhibitions in the United States and abroad, Goodnough also had solo exhibitions in 1969 at the Whitney Museum of American Art in NYC and the Albright-Knox Art Gallery in Buffalo.', ' In later years his paintings were also associated with the Color Field movement.']], ['National Student Educational Fund', ['The National Student Educational Fund (NSEF) was founded in the 1970s in Washington, D.C. as a non-profit research group.', ' Founder Layton Olson created the group using staff from the National Student Lobby (NSL) and the National Student Association (NSA).', ' NSEF issued several reports and books on issues related to students and post-secondary education.', ' Later, after NSA and NSL merged into the United States Student Association (USSA), NSEF was absorbed into USSA.']], ['The Jacksons Story', ['The Jacksons Story, an album released on Hip-O Select/UTV Records on August 28, 2007, covers fourteen years of hit singles scored by the R&B group Jacksons between 1969 and 1982.', \" It not only covers the quintet's (or sextet's) early years as the princes of Motown during the late 1960s and early 1970s, their disco innovations in the mid-1970s while still in Motown (1974) and their later funk/disco period while recording for CBS Records and its two divisions: Philadelphia International (197–1977) and Epic (1977–1981) during the late 1970s and early 1980s.\", ' For further historical value, the collection also includes solo hit singles by the sole two Jackson brothers who scored solo success including Jermaine\\'s 1979 smash, \"Let\\'s Get Serious\" and Michael\\'s hits spanning from 1971\\'s \"Got to Be There\", 1972\\'s \"Rockin\\' Robin\" and \"Ben\", 1979\\'s \"Don\\'t Stop \\'til You Get Enough\" and 1983\\'s legendary \"Billie Jean\".']], ['Get Off (Foxy album)', ['Get Off is the second album by Miami, Florida Latin Dance/Disco group Foxy.', ' Album contains Top Ten Hit \"Get Off\".']], ['The Muffins', ['The Muffins are an American Maryland-based progressive rock/avant-jazz group.', ' They were formed in Washington, DC in the early 1970s and recorded four albums before disbanding in 1981.', ' In 1998 the group reformed and recorded a further five albums and a DVD.', ' The Muffins played at Symphony Space on Broadway in NYC with Marion Brown in 1979, and also performed at a number of festivals, starting with the ZU Manifestival in New York City in 1978, The Villa Celimontana festival in Rome, Italy in 2000, two appearances at Progday in 2001 and 2002, NEARfest in 2005, and the \"Rock in Opposition\" festival in France in 2009.', ' In 2010, The Muffins headlined at Progday, making a third appearance at this long running festival.']], ['Dark Star (song)', ['\"Dark Star\" is a song released as a single by the Grateful Dead on Warner Bros. records in 1968.', ' It was written by lyricist Robert Hunter and composed by lead guitarist Jerry Garcia; however, compositional credit is sometimes extended to include Phil Lesh, Bill Kreutzmann, Mickey Hart, Ron \"Pigpen\" McKernan, and Bob Weir.', ' \"Dark Star\" was an early Grateful Dead classic and became one of their most loved and anticipated numbers, often with the group using it as a vehicle for musical improvisation sessions that extended beyond the original structure of the song.', \" The song is included in The Rock and Roll Hall of Fame's 500 Songs that Shaped Rock and Roll list and was ranked at number 57 on Rolling Stone's 100 Greatest Guitar Songs of All Time.\", ' \"Dark Star\" was often the basis for jamming during the Dead\\'s live shows, allowing the band to employ techniques typical of improvisational jazz.']], ['Go Grrrls', ['Go Grrrls is a gender-specific intervention curriculum for early adolescent girls that tries to promote a positive transition to adulthood.', ' It is a social skills building and psychoeducational program administered in a group setting—targeted towards girls in their early teens.', \" When compared to a control group using a self-reported evaluation, the program has shown a positive effect on girls' self-efficacy, body image and assertiveness.\", ' A pilot program was launched in 1995 and a final version was published in 1999.', ' It is administered by a team of two or more co-facilitators.', ' The program was designed by Craig LeCroy and Janice Daley.', ' LeCroy also published an experimental evaluation of the program.']], ['Ranch-style house', ['Ranch (also known as; American ranch, California ranch, rambler, or rancher) is a domestic architectural style originating in the United States.', ' The ranch house is noted for its long, close-to-the-ground profile, and wide open layout.', ' The house style fused modernist ideas and styles with notions of the American Western period of wide open spaces to create a very informal and casual living style.', ' While the original style of the ranch was very informal and basic in design, starting around the early 1960s, many ranch homes constructed in the United States (particularly in the Sun Belt region) were increasingly built with more dramatic features like varying roof lines, cathedral ceilings, sunken living rooms, and extensive landscaping and grounds.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.627\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae785685542994a481bbd47', 'answer': 'Oberschleißheim', 'question': 'Reima Karppinen won a silver medal in the double sculls at the Rowing Championships held in what German city?', 'supporting_facts': [['Reima Karppinen', 0], ['Reima Karppinen', 1], ['1981 World Rowing Championships', 0]], 'context': [['Kjetil Borch', ['Kjetil Borch (born 14 February 1990 in Tønsberg) is a Norwegian rower.', ' His career highlights include finishing third at the 2009 World Rowing U23 Championships with Truls Albert in bow seat.', ' Together with Nils Jakob Hoff he was fourth in the double sculls event at the 2010 World Rowing Championships and gold at the 2013 World Rowing Championships.']], ['Andrew Campbell (rower)', ['Andrew Campbell, born February 2, 1992, is an American rower who placed third in the Lightweight Single at the 2012 World Rowing Championship and fourth in the Lightweight Single at the 2011 World Rowing Championships.', ' He is also a Bronze medalist both in the single at the 2010 Junior World Rowing Championships and in the lightweight single at the 2011 Under-23 World Rowing Championships.', ' Both finishes were the highest ever for the United States in their respective events.', ' He also competed at the Under-23 World Rowing Championships in 2009 in the lightweight double and finished 17th.', \" Campbell missed qualification for the 2012 Olympic Games in the Men's Lightweight Double Sculls, placing 3rd at the 2012 Final Olympic Qualification Regatta.\", ' He graduated from Harvard University in 2014.']], ['Joachim Böhmer', ['Hans-Joachim Böhmer (1 October 1940 – 28 December 1999) was an East German rower who won a bronze medal in the double sculls at the 1972 Summer Olympics, together with Hans-Ulrich Schmied.', ' They also won a European title in 1971 and a silver medal at the 1970 World Rowing Championships.', ' In other rowing events Böhmer won a bronze medal in the eights at the 1966 World Rowing Championships.']], ['Akiko Iwamoto', ['Akiko Iwamoto (岩本 亜希子 , Iwamoto Akiko , born September 25, 1978) is a Japanese rower.', ' She was born in Nagano Prefecture.', \" She competed in the Women's lightweight double sculls at the 2012 Summer Olympics, reaching the semi-finals with her teammate Atsumi Fukumoto and ranking 12th overall.\", ' She competed in the same event at the 2000, 2004 and 2008 Summer Olympics, finishing 9th, 14th and 13th respectively.', \" She won a silver medal in the Women's double sculls at the 2002 Asian Games.\", \" She also won a silver medal in the Women's lightweight double sculls at the 2006 Asian Games, and another silver medal in the Women's lightweight double sculls at the 2010 Asian Games.\"]], ['Reima Karppinen', ['Reima Juhani Karppinen (born 27 January 1958) is a retired Finnish rower who specialized in the double sculls.', ' In this event, he won a silver medal at the 1981 World Rowing Championships, together with his legendary brother Pertti.', ' He competed at the 1984, 1988 and 1992 Summer Olympics, with other partners, and finished in 8th, 12th and 13th place, respectively.']], ['Stany Delayre', ['Stany Delayre (born 26 October 1987 in Bergerac) is a French rower.', \" At the 2012 Summer Olympics, he competed with Jérémie Azou in the men's lightweight double sculls, finishing in 4th place.\", ' On home water, he and Azou won the 2015 World Championship in that event.', ' Their team also won the silver medal at the 2014 World Championships, and won the 2013, 2014 and 2015 European Championships.', \" In 2009, Delayre was part of the French men's lightweight quadruple sculls time at the World Championships.\", \" He was also part of the French under-23s men's lightweight quadruple sculls at the 2006 and 2007 Junior World Rowing Championships.\"]], ['Jason Osborne', ['Jason Osborne (born 20 March 1994) is a German rower.', \" He won silver as part of the German team in the lightweight men's quadruple sculls at the 2013 World Rowing Championships in Chungju, Korea.\", ' He has also won medals in a number of competitions in the World Rowing Cup and European Championships.', \" He competed in the men's lightweight double sculls event at the 2016 Summer Olympics.\"]], ['1981 World Rowing Championships', ['The 1981 World Rowing Championships were World Rowing Championships that were held from 30 August to 6 September 1981 at Oberschleißheim outside Munich, Germany.']], ['Nathan Cohen (rower)', ['Nathan Phillip Cohen MNZM (born 2 January 1986) is a New Zealand rower.', ' He is a two-time world champion, and won a gold medal in the Olympics.', ' In 2006, rowing a single scull, he won a gold medal at the World University Games.', ' In doing so, he became the first New Zealander to win a gold medal at the World University Games in any sport.', \" Cohen and his rowing partner, Joseph Sullivan, won back-to-back gold medals in the men's double sculls at both the 2010 and 2011 World Rowing Championships.\", \" At the 2012 Summer Olympics, he and his partner won the gold medal in the men's double sculls, after breaking the Olympic best time in the heats.\", ' In 2013, Cohen was made a Member of the New Zealand Order of Merit for his services to rowing.']], ['Jüri Jaanson', ['Jüri Jaanson (born 14 October 1965 in Tartu) is the most successful Estonian rower of all time and the winner of five medals at World Rowing Championships.', ' He became World Champion in Tasmania 1990 in the single sculls event.', ' 14 years later, at age 38 he won an Olympic silver medal in the single sculls event at the 2004 Summer Olympics in Athens.', ' In Beijing 2008 he won his second Olympic silver medal, this time in the double sculls event with Tõnu Endrekson.', ' He is a member of the SK Pärnu rowing club located in Pärnu.', ' In 2007, Jaanson became the oldest rower ever to win a World Cup event at the age of 41 in Amsterdam.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.628\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8c6342554299240d9c214a', 'answer': 'Kelly Bundy', 'question': 'Claudine\\'s Return starred the actress who played which role on \"Married...with Children\"?', 'supporting_facts': [[\"Claudine's Return\", 0], ['Christina Applegate', 0]], 'context': [['Pierre Gaspard-Huit', ['Pierre Gaspard-Huit (29 November 1917 – 1 May 2017) was a French film director and screenwriter.', ' He directed the 1963 film \"Shéhérazade\", which starred Anna Karina.', ' He was once married to actress Claudine Auger when she was 18, and he was 43 years old.', ' She acted in several of his films.']], ['Alfred Rode', ['Alfred Rode (4 June 1905 – 22 July 1979) was an Italian-born French composer, musician, actor and film director.', ' He was born in Torre del Greco as Alfred Spedaliere.', ' In 1936 Rode appeared in the British film \"Gypsy Melody\" alongside Lupe Velez, which was a remake of his own 1935 film \"Juanita\".', ' Rode was married to the French actress Claudine Dupuis from 1951.']], ['Franchesca Salcedo', ['Franchesca \"Cruzita\" Salcedo (born Franchesca Salcedo on March 16, 2002 in San Pablo City, Laguna, Philippines) is a Filipina child actress .', ' She plays the title role of Cruzita Aldama Santibañez in \"MariMar\" daughter of Marimar Aldama and Sergio Santibañez.', ' Although her nickname in real life is Cruzita, her acting roles in both \"MariMar\" and \"Claudine\" had her playing a character named Cruzita, which she was given after her stint on the former.']], ['Claudine Auger', ['Claudine Auger (born Claudine Oger; 26 April 1941) is a French actress best known for her role as Bond girl Dominique \"Domino\" Derval in the James Bond film \"Thunderball\" (1965).', ' She earned the title of Miss France Monde and was also the first runner-up in the 1958 Miss World contest.']], ['Claudine Dupuis', ['Claudine Dupuis (born Andrée Esther Chaloum, 1 May 1924 in Paris – 26 May 1994 in Lisieux) was a French actress.', ' She starred as the \"garrulous prostitute Manon\" in Henri-Georges Clouzot\\'s \"Quai des Orfèvres\" in 1947.', ' Other films include \"The Fighting Men\" (1950), \"Les pépées font la loi\" (1954), \"Les pépées font la loi\" (1955), \"La fierecilla domada\" (1956) and \"Cuatro en la frontera\" (1958).', ' She was married to Alfred Rode.']], ['Bringing Up Bates', ['Bringing Up Bates is an American reality television show on Up TV.', ' It is centered around Gil and Kelly Jo Bates and their 19 children.', ' Gil and Kelly Jo got married on December 19, 1987, when he was 22 and she was 21.', ' Since then, they have had 9 boys and 10 girls, all of whom were born between the years 1988 and 2012, and Kelly Jo delivered every one of them.', ' There are no sets of multiples in their family either.', ' They have four children that are married: Zach (married Whitney Perkins), Michaella (married Brandon Keilen), Erin (married Chad Paine), and Alyssa (married John Webster).', \" Gil and Kelly Jo also have six grandchildren, two being Zach & Whitney's children, two being Chad & Erin's children, and the other two being John & Alyssa's children.\", ' The Bates family had a TV show in 2012 called \"United Bates of America\", and it was announced in October 2014 that the Bates family would return in a new series which would be called \"Bringing Up Bates\".', ' The series debuted on January 1, 2015.', ' UP TV revealed that the show would be returning for another season in June 2015.', ' The second season started on June 4, 2015.', ' The third season started on January 7, 2016 The fourth season started on June 2, 2016.', ' The fifth season started on January 5, 2017.', ' The sixth season began on June 1, 2017.']], ['Mandy Richardson', ['Mandy Richardson (also Hutchinson) is a fictional character from the British Channel 4 soap opera, \"Hollyoaks\", played by Sarah Jayne Dunn.', ' She debuted on-screen on 7 October 1996 and has been involved in such storylines including dealing with sexual abuse while she was a child by her father Dennis (David McAllister) and numerous failed relationships, the suicide of her brother Lewis, an on and off relationship with Tony Hutchinson (Nick Pickard) before the couple married.', ' Mandy and Tony had a daughter together who they named Grace, only for her to die from Sudden Infant Death Syndrome.', \" This led to the character and Dunn's exit from the serial in 2006.\", ' Dunn made a brief return in 2007 before making a return for six months in 2008.', ' Dunn again returned as Mandy in 2010 in a storyline which also saw the return of Warren Fox (Jamie Lomas).', ' In September 2011, Dunn announced her departure from the show and Mandy made her last appearance on 2 September 2011 before departing off-screen.', ' Dunn later returned to her role in the sixth series of \"Hollyoaks Later\" in October 2013.', ' In June 2017, it was announced that Dunn had reprised the role again and that Mandy would appear from July along with Luke Morgan played by Gary Lucy.', ' Mandy returned on 26 July 2017.']], ['Angela Lonsdale', ['Angela Lonsdale (born Angela Smith; 1970), is an English actress.', \" Born to a policeman father, Lonsdale's passion for acting was showcased in the Brewery Youth Theatre at the Brewery Arts Centre, Kendal.\", \" Working behind the box office, Lonsdale's talent was nurtured by the then Arts Centre Director, Anne Pierson.\", ' She took part in a large number of amateur productions, including plays by local playwrights John Newman-Holden and Tim Bull.', ' After initial rejection, Lonsdale then graduated from the Royal Scottish Academy of Music and Drama.', ' Lonsdale is best known for playing police officer Emma Taylor on \"Coronation Street\".', ' Taylor married veteran character Curly Watts, played by Kevin Kennedy.', ' After birth of their child, both characters left the programme in 2003.', ' She then took a regular part in the long-running television series \"The Bill\".', ' Lonsdale appeared as DI Eva Moore in the daytime BBC series \"Doctors\".', \" She left on 21 October 2008 after being shot and presumed dead by an old criminal acquaintance, but in actual reality left Leatherbridge for her own and Jimmi's safety.\", ' She made a brief return to \"Doctors\" in September 2011.', ' In 2012 and 2013 Lonsdale played the role of the mother in a family of wolves in children\\'s TV drama \"Wolfblood\".', ' Before they agreed on separation in 2010, Lonsdale was married to actor Perry Fenwick, who plays Billy Mitchell in \"EastEnders\".']], ['Christina Applegate', ['Christina Applegate (born November 25, 1971) is an American actress and dancer who, as an adolescent actress, started playing the role of Kelly Bundy on the Fox sitcom \"Married... with Children\" (1987–97).', ' In her adult years, Applegate established a film and television career, winning an Emmy and earning Tony and Golden Globe nominations.', ' She is also known for doing the voice of Brittany in the \"Alvin and the Chipmunks\" film series.']], [\"Claudine's Return\", [\"Claudine's Return is a movie released in 1998 starring Christina Applegate.\", ' It was filmed almost entirely on the American island of Tybee Island, Georgia with a few shots from the surrounding areas.', ' It was released as Kiss of Fire on DVD.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.629\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a72d3525542991f9a20c5ab', 'answer': '1905', 'question': \"What was the founding year of the county where Weber's Store is located?\", 'supporting_facts': [[\"Weber's Store\", 0], ['Sanders County, Montana', 3]], 'context': [['Karaj Payam Noor University', ['Karaj Payam Noor University is located in Karaj, Iran, and has two campuses.', ' The main campus is located in Gohardasht, and another campus is located on Ghalamestan Street.', ' The university was founded in 2000-01.', \" In the school's founding year, 70 students were admitted for their BS in accounting.\", ' The university now offers 52 courses at BS, BA, BE, MS, and MBA levels and has over 15,000 students.']], ['Louaize Club', ['Louaize Club is the basketball department of Notre Dame University – Louaize , a university basketball club basked in Zouk Mosbeh.', ' The club was established in the founding year of 1978 and is currently participating in the 2016 Lebanese Basketball League.']], ['Diocesan Native Female Training School', ['Diocesan Native Female Training School (DNFTS, ) was a school under the Anglican Church of Hong Kong in the 19th century, founded in 1860 and closed down in 1868.', \" Its premises now belong to today's Bonham Road Government Primary School().\", \" In 1869, another institution called Diocesan Home and Orphanage (DHO, later renamed Diocesan School and Orphanage, and now known as Diocesan Boys' School) was founded in the same place.\", ' Due to the obvious differences in founding groups, vision of education, personnel arrangement and students’ background, DNFTS has been regarded only as a forerunner, and called ‘the First Foundation’ by DHO and later DBS.', ' Using 1869 as its founding year, DBS calls itself ‘the Second Foundation’.', \" As for Diocesan Girls' School, founded in Rose Villas near DSO in 1899, it claims to be the successor of DNFTS and traces the founding year back to 1860.\"]], ['Sanders County, Montana', ['Sanders County is a county located in the U.S. state of Montana.', ' As of the 2010 census, the population was 11,413.', ' Its county seat is Thompson Falls.', ' The county was founded in 1905.']], ['Gufo Temple', ['Gufo Temple () is located on the bank of Qingshui River, Jingangku, Shanxi province, China and is the first temple to see if entering the Mount Wutai area from the south route.', \" According to Mount Wutai's history, there are only records of the renovation of this temple, but nothing concerning its founding year.\", ' Thus, it is speculated \"Old Buddha exists before the beginning of the world.', ' Gufo Temple (Old Buddha Temple) exists before Mount Wutai.\"']], ['Muscular Dystrophy Canada', ['Muscular Dystrophy Canada (MDC) (French: Dystrophie musculaire Canada ) is a non-profit organization that strives to find a cure for neuromuscular disorders.', ' Founded in 1954 as Muscular Dystrophy Association of Canada, volunteers and staff nationwide have helped to provide support and resources to those affected.', ' Since the founding year, over $64 million has been put towards research via collaborations, fundraising events, and donations.']], ['Symphony of Southeast Texas', ['The Symphony of Southeast Texas is an American orchestra based in Beaumont, Texas.', ' The orchestra, formerly known as the \"Beaumont Symphony Orchestra\", officially started in 1953; however, the impetus can be traced back as early as 1923 with the formation of the Beaumont Music Commission.', ' The 2015-16 season is the sixty-third consecutive season since the founding year.', \" The symphony's home theater is the Julie Rogers Theater in downtown Beaumont.\", ' The symphony lists over eighty musicians in the orchestra as of 2015.']], ['Stavanger', ['Stavanger (] ) is a city and municipality in Norway.', ' The city is the third-largest urban zone and metropolitan area in Norway (through conurbation with neighbouring Sandnes) and the administrative centre of Rogaland county.', ' The municipality is the fourth most populous in Norway.', ' Located on the Stavanger Peninsula in Southwest Norway, Stavanger counts its official founding year as 1125, the year the Stavanger Cathedral was completed.', \" Stavangers core is to a large degree 18th- and 19th-century wooden houses that are protected and considered part of the city's cultural heritage.\", \" This has caused the town centre and inner city to retain a small-town character with an unusually high ratio of detached houses, and has contributed significantly to spreading the city's population growth to outlying parts of Greater Stavanger.\"]], ['Skultuna mässingsbruk', ['Skultuna Messingsbruk is a Swedish company founded in 1607 at the bequest of King Karl IX.', ' Skultuna Messingsbruk is located in Skultuna on the outskirts of Västerås.', ' The logotype of Skultuna consists of the closed royal crown, the name \"Skultuna\" and the founding year \"1607\".']], [\"Weber's Store\", [\"Weber's Store, at 510 Main St. in Thompson Falls in Sanders County, Montana was listed on the National Register of Historic Places in 1986.\", ' It has also been known as Thompson Falls Laundry.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.630\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac3d2ac554299741d48a330', 'answer': 'comedian', 'question': 'What has Henry Winkler worked as that Norman Foster has not?', 'supporting_facts': [['Henry Winkler', 0], ['Norman Foster (director)', 0]], 'context': [['Cop and a Half', ['Cop and a Half is a 1993 American buddy cop-comedy film directed by Henry Winkler, and stars Burt Reynolds, Norman D. Golden II, and Ray Sharkey in his final role.', ' Reynolds plays a veteran cop who reluctantly takes an eight-year-old child (Golden) as his partner to solve a murder investigation.']], ['Team 4', ['Team 4 was a British architectural firm, established in 1963 by architecture graduates Su Brumwell, Wendy Cheesman, Norman Foster and Richard Rogers.', ' Friction emerged within the firm, and by June 1967, Foster and Rogers, decided to dissolve the firm.']], ['Henry Winkler', ['Henry Franklin Winkler (born October 30, 1945) is an American actor, director, comedian, producer, and author.']], ['List of Hank Zipzer episodes', ['\"Hank Zipzer\" is a children\\'s television series which stars Nick James in the titular role as a 12-year-old dyslexic schoolboy.', \" The show is based on the series of books by Henry Winkler, who plays the character of Mr. Rock, Hank's music teacher.\", ' The first series premiered in January 2014 on CBBC and a second and third series have been commissioned.', ' Unlike the books that took place in America, the series takes place in Britain.', ' The second series began airing on 13 August 2015.', ' Javone Prince made his first appearance as Mr Joy in series 2, episode 5, \"Hank\\'s Hero\".', ' The third series began airing on 26 May 2016, which was followed by an 84-minute Christmas movie on 12 December 2016.', ' A fourth series has been confirmed by Winkler.']], ['Anthony Hunt', ['Anthony Hunt (born 1932) is a structural engineer of numerous world-renowned buildings, with a career spanning from the 1950s until his retirement in 2002.', ' With a strong interest in both engineering and industrial design, Hunt was a major player in creating the High Tech style of Norman Foster and Richard Rogers.', ' He formed Anthony Hunt associates in 1962.', ' He worked with Rogers and Foster on Reliance Controls building in Swindon (1966) which was the first building of the High Tech architecture style.', ' He was also a structural engineer on the Waterloo International railway station in London (1993).']], ['Night Shift (film)', ['Night Shift is a 1982 American comedy film, directed by Ron Howard, concerning a timid night shift morgue employee whose life is turned upside down by a free-spirited entrepreneur.', ' It stars Howard\\'s \"Happy Days\" co-star Henry Winkler along with Michael Keaton, in his first starring role, and Shelley Long.', ' Also appearing are Richard Belzer and Clint Howard.', ' A young Kevin Costner has a brief scene as \"Frat Boy #1\", Shannen Doherty appears as a Bluebell scout, Vincent Schiavelli plays a man who delivers a sandwich to Winkler\\'s character, and Charles Fleischer has a brief role as one of the jail prisoners.']], ['Norman Foster (director)', ['Norman Foster (born Norman Foster Hoeffer, December 13, 1903 – July 7, 1976) was an American actor, film director and screenwriter.']], ['Immanuel Winkler', ['Immanuel Winkler (June 3, 1886 in Sarata – June 18, 1932 in Winnipeg), born Adolf Immanuel Mathaeus Winkler, was a pastor in Hoffnungstal (today Tsebrykove, Ukraine) and author.', ' During World War I, Winkler worked for the rights of Germans in Russia.']], ['Langley Academy, Slough', ['The Langley Academy is an academy in Langley, east of Slough in Berkshire, south east England.', ' It opened in September 2008, replacing the former Langleywood Secondary School.', ' The building was designed by Foster and Partners, led by the architect Norman Foster and by Buro Happold.', ' The school is supported by the Arbib Foundation.', ' It has a specialism of science and a museum education theme, including exhibits in the school building.', ' It also promotes sport, notably cricket and rowing.']], ['Geoffrey Foster', ['Geoffrey Norman Foster (16 October 1884 – 11 August 1971) was an English cricketer who played county cricket for Worcestershire and Kent, as well as appearing a number of times for Oxford University and MCC.', ' He was one of the seven Foster brothers, all of whom played first-class cricket for Worcestershire, and he led the county on a few occasions in the absence of the regular captain.', ' He was a fast scorer, once making 101 in an hour for Oxford against Gentlemen of England.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.632\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a88a19a5542997e5c09a64b', 'answer': 'moth', 'question': 'The American Sweetgum is the hostplant of what kind of bug?', 'supporting_facts': [['Phyllocnistis liquidambarisella', 0], ['Phyllocnistis liquidambarisella', 1], ['Liquidambar styraciflua', 0]], 'context': [['The Love Bug (1997 film)', ['The Love Bug is a 1997 American made-for-television comedy adventure film and a sequel/remake of the 1968 film of the same name produced by Walt Disney Television which premiered on ABC as part of \"The Wonderful World of Disney\" on November 30, 1997.', ' The remake starred Bruce Campbell and included a special appearance by Dean Jones, star of the original \"The Love Bug\", tying it to the previous films and introduced an evil black Volkswagen named Horace, \"The Hate Bug\", giving the film a much darker tone than the other \"Herbie\" films.']], ['Melacoryphus lateralis', ['Melacoryphus lateralis is a species of true bug, one of several called black-and-red seed bug.', ' Black and fringed with red and gray, some call it the charcoal seed bug, due to its resemblance to a dying ember.', ' Native to the deserts of western North American, they have a tendency to appear in large numbers in the late summer.']], ['Manahawkin Wildlife Management Area', ['Manahawkin Wildlife Management Area (Manahawkin Bottomland Hardwood Forest) is a 1642 acre wildlife management area near Manahawkin, Stafford Township, Ocean County, New Jersey.', ' It was designated a National Natural Landmark in January 1976.', ' It is known for its mature bottomland hardwood forest which contains examples of American sweetgum, red maple and black gum trees.']], ['Liquidambar styraciflua', ['American sweetgum (\"Liquidambar styraciflua\"), also known as American storax, hazel pine, bilsted, redgum, satin-walnut, star-leaved gum, alligatorwood, or simply sweetgum, is a deciduous tree in the genus \"Liquidambar\" native to warm temperate areas of eastern North America and tropical montane regions of Mexico and Central America.', ' Sweet gum is one of the main valuable forest trees in the southeastern United States, and is a popular ornamental tree in temperate climates.', ' It is recognizable by the combination of its five-pointed star-shaped leaves and its hard, spiked fruits.', ' It is currently classified in the plant family Altingiaceae, but was formerly considered a member of the Hamamelidaceae.']], ['Ischnodemus sabuleti', ['Ischnodemus sabuleti, also known as the European chinch bug, is a species of swarming true bug from the family Blissidae, which family also includes the American Chinch Bug \"Blissus leucopterus\".', ' It was first described by Carl Fredrik Fallén in 1826.']], ['Stenodema laevigatum', ['Stenodema laevigatum, or sometimes Stenodema laevigata (also called Grass bug), is a carnivorous species of bug from Miridae family.', ' The species have a gray to brown elongated body, with the eyes located backwards in the head.', ' Sometimes they might come in green colour.', ' They are 8 - in length, which makes it a rather big species of its kind.', ' They are common in the United Kingdom, and throughout the rest of Europe.']], ['Datronia scutellata', ['Datronia scutellata is a plant pathogen that causes wood rot on \"Liquidambar\" (sweetgum) and \"Platanus occidentalis\" (American sycamore) trees.']], ['Green shield bug', ['The green shield bug (\"Palomena prasina\") is a shield bug of the family Pentatomidae.', ' It may also be referred to as a green stink bug, particularly outside of Britain, although the name green stink bug more appropriately belongs to the larger North American stink bug, \"Acrosternum hilare\".', ' The adult green shield bug ranges in the colour of their backs from bright green to bronze, without any substantial markings.', ' Green shield bugs are a very common shield bug throughout Europe, including the British Isles, and are found in a large variety of habitats, including gardens.', ' They have been found as far north as 63° N latitude.']], ['Phyllocnistis liquidambarisella', ['Phyllocnistis liquidambarisella is a moth of the Gracillariidae family, known from the United States (New York, Maryland, Kentucky, Georgia, Texas, Florida).', ' The hostplant for the species is \"Liquidambar styraciflua\".', ' They mine the leaves of their host plant.', ' The mine has the form of a long, winding, linear mine on the upperside of the leaf.', ' It is rather indistinct, without any central line of frass.']], ['Liquidambar', ['Liquidambar, commonly called sweetgum (sweet gum in the UK), gum, redgum, satin-walnut, or American storax, is the only genus in the flowering plant family Altingiaceae with 15 species.', ' They were formerly often treated in Hamamelidaceae.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.632\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7a3a2a5542996c55b2dd41', 'answer': '27 November 1956', 'question': 'The author of Some Other Rainbow was born on what day and year?', 'supporting_facts': [['Some Other Rainbow', 0], ['John McCarthy (journalist)', 0]], 'context': [['John McCarthy (journalist)', ['John Patrick McCarthy CBE (born 27 November 1956) is a British journalist, writer and broadcaster, and one of the hostages in the Lebanon hostage crisis.', \" McCarthy was the United Kingdom's longest-held hostage in Lebanon, where he was a prisoner for more than five years.\"]], ['Alex Sánchez (author)', ['Alex Sánchez (born 1957) is a Mexican-American author of award-winning novels for teens and adults.', ' His first novel, \"Rainbow Boys\" (2001), was selected by the American Library Association (ALA), as a Best Book for Young Adults.', ' Subsequent books have won additional awards, including the Lambda Literary Award.', \" Although Sanchez's novels are widely accepted in thousands of school and public libraries in America, they have faced a handful of challenges and efforts to ban them.\", ' In Webster, New York, removal of \"Rainbow Boys\" from the 2006 summer reading list was met by a counter-protest from students, parents, librarians, and community members resulting in the book being placed on the 2007 summer reading list.']], ['Kenneth Ascher', ['Kenneth Lee \"Kenny\" Ascher (born October 26, 1944 in Washington, D.C.) is an American jazz pianist, composer, and arranger who is active in jazz, rock, classical, and musical theater genres — in live venues, recording studios, and cinema production.', ' He is widely known for co-writing, with Paul Williams, \"Rainbow Connection\" from \"The Muppet Movie\".', ' Both Williams and Ascher received Oscar nominations for the 1979 Academy Awards for Best Original Song (\"Rainbow Connection\") and Best Original Score (\"The Muppet Movie\" Soundtrack).', ' The song was also nominated for the Golden Globes for \"Best Original Song\" that same year.']], ['Some Other Rainbow', ['Some Other Rainbow is a joint memoir written by John McCarthy and Jill Morrell and first published by Bantam Press in 1993.', \" It deals in separate chapters with the individual and parallel experiences of McCarthy and Morrell, during McCarthy's captivity in the Lebanon, which lasted from 17 April 1986 until 8 August 1991.\"]], ['Jennifer Gillis', ['Jennifer Kristine Gillis (born 26 April 1996) is a Canadian singer, dancer and actress with an extensive resume in musical theatre, television, singing, radio, recording, and animated voice-over work.', ' Gillis is most noted for performing in Sir Andrew Lloyd Webber\\'s CBC TV reality show \"Over the Rainbow\"-a competition to be cast as the leading role of Dorothy Gale in Lloyd Webber\\'s forthcoming production of \"The Wizard of Oz\" in Toronto, Ontario.', ' Being the youngest aspiring singer in the competition, she singularly represented her province of British Columbia and as a result was named the Top 6th musical theatre performer in all of Canada.', ' Since \"Over the Rainbow\", Gillis sang the Canadian national anthem \"O Canada\" for the Prime Minister of Canada, Stephen Harper on Parliament Hill in Ottawa, Ontario on Canada Day in 2013.', ' She aspires to become a Broadway actress someday and wants to dedicate her life to performing.']], ['Alberto Ruz Buenfil', ['Alberto Ruz Buenfil (born 1945) is a native of Mexico whose work is dedicated to social change, environmental sustainability, and the performing arts.', \" He co-founded two international theater groups as well as Mexico's first ecovillage, known as Huehuecoyotl.\", ' He led the 13-year Rainbow Peace Caravan, an international effort to promote sustainable design and permaculture, as well as theatrical performances, across seventeen countries of Latin America.', ' He was also funded by Ashoka from 2002 to 2005, and received in the name of the Rainbow Peace Caravan, the prize \"Escuela Viva\" from the Brazilian President Lula da Silva and Minister of Culture Gilberto Gil, as one of the 60 most advanced projects in education in the country.']], ['David Robie', ['David Robie (born 1945) is a New Zealand author, journalist and media educator who has covered the Asia-Pacific region for international media for more than two decades.', ' He became an associate professor in Auckland University of Technology School of Communication Studies in 2005 and a professor in 2011.', ' In 1985, Dr Robie sailed on board the Greenpeace eco-navy flagship \"Rainbow Warrior\" for 10 weeks until it was bombed by French secret agents in New Zealand’s Auckland harbour.', ' He is the author of a book about the ill-fated voyage, \"Eyes of Fire: The Last Voyage of the Rainbow Warrior\" (Lindon Books, 1986).', ' An updated memorial edition of \"Eyes of Fire\" was published in July 2005 and a 30th anniversary edition in July 2015 (Little Island Press).']], ['José Argüelles', ['José Argüelles, born Joseph Anthony Arguelles ( ; January 24, 1939 – March 23, 2011), was an American New Age author and artist.', ' He was the founder of Planet Art Network and the Foundation for the Law of Time.', ' He held a Ph.D. in Art History and Aesthetics from the University of Chicago and taught at numerous colleges, including Princeton University, the University of California, Davis, the San Francisco Art Institute, and Evergreen State College.', ' As one of the originators of the Earth Day concept (due in part to the influence of astrologer Dane Rudhyar), Argüelles founded the first Whole Earth Festival in 1970, at Davis, California.', ' He is best known for his leading role in organizing the 1987 Harmonic Convergence event, for inventing (with the assistance of his wife Lloydine) the perpetual Dreamspell calendar in 1992, and for the central role that he played in the emergence of the 2012 phenomenon.', ' Towards the end of his life, Argüelles focused on issue of consciousness, elaborating the concept of a noosphere (based on the work of Teilhard de Chardin and Vladimir Vernadsky) as a global work of art.', ' Specifically, he envisioned a \"rainbow bridge\" encircling the Earth.']], ['Manitonquat', ['Manitonquat (AKA Medicine Story; born Francis Story Talbot, July 17, 1929), is an American author of two books, and several more self-published booklets on New Age philosophy, spirituality, and community sociology.', ' He has led workshops and rituals at the Rainbow Gatherings held by the Rainbow Family.', ' Since the 1970s, he has toured the United States and Europe teaching and lecturing on the adaptation of the application of philosophy to modern problems of society, community, and relationships.']], ['Joe Lynn Turner', ['Joe Lynn Turner (born Joseph Arthur Mark Linquito, August 2, 1951) is an American singer, guitarist, songwriter, and producer.', ' He is known for his work in the hard rock bands Rainbow and Deep Purple.', ' During his career, Turner fronted and played guitar with pop rock band Fandango in the late 1970s; and in the early 80s, he became a member of Rainbow, fronting the band and writing songs with guitarist, Ritchie Blackmore and bassist, and producer, Roger Glover.', ' After Rainbow had disbanded (the first time) in March 1984, he pursued a solo career, released one album, Rescue You, and then later did session work, singing background vocals for the likes of Billy Joel, Cher, and Michael Bolton.', ' On the advice of Bolton, Turner began recording jingles for radio and television.', ' Other songs he had composed or through collaboration with songwriters like Desmond Child and Jack Ponti were being recorded and released by international recording artists Jimmy Barnes, Lee Aaron, and Bonfire.', ' Turner had a short-lived association with neoclassical metal guitarist Yngwie Malmsteen and then Deep Purple.', ' From the mid-1990s, he resumed his solo career, releasing an additional nine studio and two live recordings.', \" Turner did other session work, appearing as lead vocalist on tribute albums and working on projects involving various musical groups including progressive rock band Mother's Army; Bulgarian hard rock band Brazen Abbot; funk rock duo Hughes Turner Project; and classic rock/ progressive rock band Rated X.\", ' In 2006, Frontiers Records approached Turner to become involved with the AOR side project Sunstorm.', ' By 2016, four albums under the Sunstorm name had been released.', ' That same year, Turner released \"The Sessions\" via Cleopatra Records featuring a veritable who\\'s who of classic rock royalty as guest musicians, before resuming his seemingly constant touring schedule back in Europe']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.633\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab572ff5542992aa134a31b', 'answer': 'Louis King', 'question': 'Who directed the film which starred Steve Cochran and the actor who played Sheriff Roy Coffee in \"Bonanza\"?', 'supporting_facts': [['The Lion and the Horse', 0], ['The Lion and the Horse', 1], ['Ray Teal', 1]], 'context': [['Rockmond Dunbar', ['Rockmond Dunbar (born January 11, 1973 in Berkeley, California) is an American actor.', ' He is best known for his roles as Baines on the NBC series \"Earth 2\", Kenny Chadway on Showtime\\'s \"Soul Food\", and Benjamin Miles \"C-Note\" Franklin on the FOX crime drama \"Prison Break\".', ' He also played Sheriff Eli Roosevelt on the FX Drama series \"Sons of Anarchy\", FBI Agent Dennis Abbott on \"The Mentalist\", and FBI Agent Abe Gaines in the Hulu series \"The Path\".']], ['Storm Warning (1951 film)', ['Storm Warning is a 1951 American film noir thriller, directed by Stuart Heisler, and featuring Ginger Rogers, Ronald Reagan, Doris Day and Steve Cochran.', ' Lauren Bacall was originally cast in the part eventually played by Rogers.', ' Bacall turned it down and was put on suspension by Warner Bros. for her defiance.']], ['Ray Teal', ['Ray Teal (January 12, 1902April 2, 1976) was an American actor who appeared in more than 250 films and some 90 television programs in his 37-year career.', ' His longest-running role was as Sheriff Roy Coffee on NBC\\'s western series \"Bonanza\" (1960–1972).', ' He also played a sheriff in the film \"Ace in the Hole\" (1951).']], ['Operation Secret', ['Operation Secret is a 1952 American drama film directed by Lewis Seiler and written by Harold Medford and James R. Webb.', ' The film stars Cornel Wilde, Steve Cochran, Phyllis Thaxter, Karl Malden, Paul Picerni and Lester Matthews.', ' The film was released by Warner Bros. on November 8, 1952.', ' The film is based on the exploit of US Marine Corps Major Peter Ortiz.']], ['The Lion and the Horse', ['The Lion and the Horse is a 1952 Western film directed by Louis King and written by Crane Wilbur.', ' It stars Steve Cochran, Ray Teal and a horse named Wildfire.']], ['Steve Cochran', ['Steve Cochran (May 25, 1917 - June 15, 1965) was an American film, television and stage actor.', ' He graduated from the University of Wyoming in 1939.', ' After a stint working as a cowpuncher, Cochran developed his acting skills in local theatre and gradually progressed to Broadway, film and television.']], ['Alfred Sandor', ['Alfred Sandor (November 5, 1918 – September 22, 1983), born as Alfred Sandwina, and billed early in his career as Al Sandwina was a Hungarian-born American and Australian character actor and singer, born in Budapest during the turmoil of the Hungarian Revolutions and Interventions, He spent his early years travelling with the circus, where his mother, Katie Sandwina, was a circus strongwoman, and he had a background as a ringmaster.', ' After a brief career working as a boxer in New York City, he found himself working as a spy behind enemy lines during World War II, for the Counter Intelligence Agency of the US Army.', ' Having returned to America, he established himself as an actor, appearing in Broadway Productions and Musician Theatre.', ' On US television he appeared on the Phil Silvers show and played Sheriff George Patterson in a single episode of \"Dark Shadows\" in 1968 and on Our Five Daughters, theatre work included Neil Simon\\'s \"The Odd Couple\", and Gypsy opposite Ethel Merman, film work included \"The Return of Captain Invincible\".']], ['Il Grido', ['Il grido (English: \"The Cry\" ) is a 1957 Italian black-and-white drama film directed by Michelangelo Antonioni and starring Steve Cochran, Alida Valli, Betsy Blair, and Dorian Gray.', ' Based on a story by Antonioni, the film is about a man who wanders aimlessly, away from his town, away from the woman he loved, and becomes emotionally and socially inactive.', ' \"Il Grido\" won the Locarno International Film Festival Golden Leopard Award in 1957, and the Italian National Syndicate of Film Journalists Silver Ribbon Award for Best Cinematography (Gianni di Venanzo) in 1958.']], ['Ewing Mitchell', ['Ewing Young Mitchell (December 29, 1910 – September 3, 1988) was an American character actor of film and television best known for his role as Sheriff Mitch Hargrove in 26 episodes between 1956 and 1959 of the aviation adventure series with a western theme, \"Sky King\".', ' He also played Sheriff Powers on another western series, \"The Adventures of Champion\".']], [\"The Damned Don't Cry\", [\"The Damned Don't Cry is a 1950 American film noir crime-drama directed by Vincent Sherman and featuring Joan Crawford, David Brian, and Steve Cochran.\", \" It tells of a woman's involvement with an organized crime boss and his subordinates.\", ' The screenplay by Harold Medford and Jerome Weidman was based on the story \"Case History\" by Gertrude Walker.', ' The plot is loosely based on the relationship of  Bugsy Siegel and Virginia Hill.', ' The film was directed by Vincent Sherman and produced by Jerry Wald.', ' \"The Damned Don\\'t Cry!\"', ' is the first of three cinematic collaborations between Sherman and Crawford, the others being \"Harriet Craig\" (1950) and \"Goodbye, My Fancy\" (1951).']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.633\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8b8f5b55429949d91db5e1', 'answer': 'video game', 'question': 'What form of play does Yameen and Activision have in common?', 'supporting_facts': [['Yameen', 2], ['Activision', 0]], 'context': [['Activision', ['Activision Publishing, Inc., also known as Activision, is an American video game publisher.', \" It was founded on October 1, 1979 and was the world's first independent developer and distributor of video games for gaming consoles.\", ' Its first products were cartridges for the Atari 2600 video console system published from July 1980 for the US market and from August 1981 for the international market (UK).']], ['Raven Software', ['Raven Software (or Raven Entertainment Software, Inc.) is an American video game developing company based in Wisconsin and founded in 1990.', ' In 1997, Raven made an exclusive publishing deal with Activision and was subsequently acquired by them.', ' After the acquisition, many of the studio\\'s original developers, largely responsible for creating the \"Heretic\" and \"\" games, left to form Human Head Studios.']], ['The Legend of Spyro: Dawn of the Dragon', ['The Legend of Spyro: Dawn of the Dragon is the third and final installment in \"The Legend of Spyro\" trilogy, as well as the tenth anniversary game of the series.', ' It was developed by Étranges Libellules and published by Activision in North America and Sierra Entertainment in International for the PlayStation 2, PlayStation 3, Wii and Xbox 360.', ' Tantalus Media developed the Nintendo DS version.', ' It is the end of the second Spyro continuity, with \"\" serving as the second reboot of the franchise after Vivendi Games merged with video game publisher Activision to form the Activision Blizzard holding company on July 9, 2008.']], ['Guitar Hero 5', ['Guitar Hero 5 (initially referred to as Guitar Hero V) is a music rhythm game and the fifth main entry in the \"Guitar Hero\" series.', ' The game was developed by Neversoft and published by Activision, and released internationally in September 2009 for the Xbox 360, PlayStation 2, 3 and Wii consoles.', ' Similar to the preceding title, \"Guitar Hero World Tour\", \"Guitar Hero 5\" is geared towards playing in a four-person band experience, including lead and bass guitar, drums, and vocals.', ' The game is available as a standalone title, allowing players to use existing compatible instrument controllers, and as a bundle that provides these controllers.', ' \"Guitar Hero 5\" adds several new features, such as drop-in/drop-out play, bands composed of any combination of available instruments, a Rockfest competitive mode consisting of several various scoring mechanisms, and both song-specific and general Challenges to unlock new avatars, clothing, and other extras in the game.', ' Many of these changes were added to make the game a more social experience, allowing players across a range of skill levels to be able to play cooperatively and competitively against each other both locally and online.']], ['Right Back at It Again', ['\"Right Back at It Again\" is the second track and the first single from A Day to Remember\\'s fifth album, \"Common Courtesy\" (2013).', ' In October 20, 2015, the song was featured in Activision rhythm-music game, \"\".']], ['Yameen', ['Yameen is a hiphop producer from Philadelphia, PA.', ' His most recent work, \"Come On & Go Off\" was released on September 2nd, 2014 on Rumble Pack Records.', ' His music can be heard weekly on the Activision videogames podcast, One of Swords where he is also occasionally a guest commentator.']], ['Intellivision Rocks', ['Intellivision Rocks is the PC-only sequel to the original PC version of \"Intellivision Lives!', '\".', ' As with \"Intellivision Lives!\"', ', \"Intellivision Rocks\" is a collection of games which were originally found on the Intellivision, presented in emulated form.', ' It mainly features 3rd-party games from Activision and Imagic.', ' In addition, several unreleased games are included.']], ['Form 8-K', ['Form 8-K is a very broad form used to notify investors in United States public companies of specified events that may be important to shareholders or the United States Securities and Exchange Commission.', ' This is one of the most common types of forms filed with the SEC.', ' After a significant event like bankruptcy or departure of a CEO, a public company generally must file a Current Report on Form 8-K within four business days to provide an update to previously filed quarterly reports on Form 10-Q and/or Annual Reports on Form 10-K.', ' Form 8-K is required to be filed by public companies with the SEC pursuant to the Securities Exchange Act of 1934, as amended.', ' For a list of events that would trigger a Form 8-K to be filed, see the Official SEC Form 8-K Summary, briefly below, and this fully annotated Form 8-K, which contains links to all rules and SEC guidance applicable to the form.']], ['Guitar Hero', ['The Guitar Hero series (sometimes referred to as the Hero series) is a series of music rhythm games first published in 2005 by RedOctane and Harmonix, and distributed by Activision, in which players use a guitar-shaped game controller to simulate playing lead, bass guitar, and rhythm guitar across numerous rock music songs.', ' Players match notes that scroll on-screen to colored fret buttons on the controller, strumming the controller in time to the music in order to score points, and keep the virtual audience excited.', ' The games attempt to mimic many features of playing a real guitar, including the use of fast-fingering hammer-ons and pull-offs and the use of the whammy bar to alter the pitch of notes.', ' Most games support single player modes, typically a Career mode to play through all the songs in the game, and both competitive and cooperative multiplayer modes.', ' With the introduction of \"Guitar Hero World Tour\" in 2008, the game includes support for a four-player band including vocals and drums.', ' The series initially used mostly cover versions of songs created by WaveGroup Sound, but most recent titles feature soundtracks that are fully master recordings, and in some cases, special re-recordings, of the songs.', ' Later titles in the series feature support for downloadable content in the form of new songs.']], ['Lotion play', ['Lotion Play is a subset of the better known Wet-And-Messy fetish (WAM), which typically involves participants using food (such as pudding or whipped cream), mud, or paint as a lubricant to facilitate sexual activity.', ' Lotion Play isolates lotion specifically as a lubricating medium - setting it apart in the Wet-and-Messy genre, as other common WAM mediums do not have such specific popularity as Lotion Play.', ' Lotion Play (ローションプレイ , rōshon purei ) , also known as gookkake, gluekkake, is a popular fetish, form of Japanese erotica and prostitution request involving the use of copious amounts of lubricant, which in the Japanese language is referred to by the Old-French word \"lotion\" (ローション in Japanese).', 'Typically lotion play involves a participant rubbing lotion on another using their body, sexual intercourse in a pool or bath filled with lotion, or lotion being poured over the participants during sex.', ' In Japan (and other parts of the world), lotion is available in concentrated form (e.g.: liter/gallon) which can be added to hot water to produce the desired amount of lotion.', ' A 1-gallon concentrate will typically yield 6-10 gallons of lotion (J-Lube Lotion Concentrate).', ' The main component in most lotion is polyacrylate.', ' A similar effect can be achieved by dissolving powdered methyl cellulose in water.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.634\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae1c2d8554299234fd042ee', 'answer': 'in 1994', 'question': 'When did  former Yugoslav which  Patricia Hy-Boulais eventually lost to in  1992 US Open became US citizen', 'supporting_facts': [['Patricia Hy-Boulais', 5], ['Monica Seles', 0], ['Monica Seles', 1], ['Monica Seles', 2]], 'context': [['Sanjalice', ['Sanjalice (Serbian Cyrillic: Сањалице, trans.', ' \"The Dreamers\") was a former Yugoslav rock band from Belgrade, notable for being one of the first former Yugoslav all-female bands, as well as one of the pioneers of the former Yugoslav rock scene.']], ['1992 US Open (tennis)', ['The 1992 US Open was a tennis tournament played on outdoor hard courts at the USTA National Tennis Center in New York City in New York in the United States.', ' It was the 112th edition of the US Open and was held from August 31 to September 13, 1992.']], ['Smak', ['Smak (Serbian Cyrillic: Смак; trans.', ' \"The end time\") is a Serbian and former Yugoslav rock band from Kragujevac.', ' The group reached the peak of popularity in the 1970s when it was one of the most notable acts of the former Yugoslav rock scene.', ' The band\\'s leader, guitarist Radomir Mihailović, nicknamed Točak (\"The Wheel\"), is considered one of the most influential guitarists on the former Yugoslav rock scene.']], ['Patricia Hy-Boulais', ['Patricia Hy-Boulais (born 22 August 1965) is a former tennis player.', ' She turned professional on October 12, 1986.', ' Early in her career she represented Hong Kong (since the beginning until the end of the 1987 season).', ' She became a citizen of Canada in 1991.', ' However, she represented Canada just since the beginning of the 1988 season.', ' Her best performance at a Grand Slam came when she got to the quarter finals of the 1992 US Open, defeating Eva Švíglerová, Judith Wiesner, Jennifer Capriati and Helena Suková before losing to eventual champion Monica Seles.']], ['Heller (band)', ['Heller is a Serbian and former Yugoslav speed/thrash metal band from Belgrade, notable as one of the first Yugoslav thrash metal bands and one of the pioneers of former Yugoslav extreme metal.', \" The band's debut self-titled album is arguably the first extreme metal album in former Yugoslavia, and one of the first of the kind in Southeastern Europe.\"]], ['Milan Panić', ['Milan Panić (Serbian Cyrillic: , ] ); born 20 December 1929) is a Serbian American former Prime Minister of Yugoslavia, humanitarian, and multimillionaire businessman based in Newport Beach and Pasadena, California.', ' He served as Prime Minister of the Federal Republic of Yugoslavia from 1992 to 1993.', ' During and after his time as Prime Minister, he campaigned for peace and democracy in the Balkan region.', ' He ran for President of Serbia in 1992, ultimately coming in second to Slobodan Milošević in an election marked by allegations of media and vote tampering by the ruling party.', ' Panić became Prime Minister of Yugoslavia while an American citizen.', ' The legality of retaining US citizenship while accepting this office has been questioned based on a Constitutional prohibition of a US citizen accepting office on behalf of a foreign nation.', ' Panić is the first US citizen to occupy a high-level political position in a foreign country since Golda Meir.']], ['Sretno dijete', ['Sretno dijete (English: \"Happy child\") is a Croatian documentary film directed by Igor Mirković, and produced by Rajko Grlić, an Ohio University professor of film, in 2003.', ' The film is a nostalgic autobiographical overview of the authors adolescence in SR Croatia in the former Socialist Federal Republic of Yugoslavia during the late 1970s and early 1980s which corresponded with the emergence of the Yugoslav punk rock and new wave scenes, both which the author affiliated to, thus turning this film into a rockumentary.', \" The film features interviews and rare footage of some of the top former Yugoslav rock acts ever such as: Azra, Film and Haustor from the author's hometown Zagreb, Croatia where most of the story takes place; then members of Električni orgazam and Idoli whom the author visits in Belgrade, Serbia; as well as Pankrti and Buldožer from Ljubljana, Slovenia.\", ' Beside materials filmed around former Yugoslavia, the film also contains interviews with important former Yugoslav artists who currently live abroad.', ' For example, Darko Rundek is interviewed in Paris, France, Mirko Ilić in New York City in the United States, and there are also scenes shot on locations in the Netherlands, Germany, Hungary and other countries.', ' The film is named after a song by Prljavo kazalište from their first self-titled album.']], ['Ludmila Richterová', ['Ludmila Richterová (born 7 March 1977) is a Czech former tennis player.', ' She reached her highest ranking, World No. 62, on 18 March 1996, and won one WTA Tour title, the 1995 Rover British Clay Court Championships in Bournemouth, England, by beating Patricia Hy-Boulais 6(10)–7, 6–4, 6–3.', ' In her career, Richterová defeated players such as Barbara Schett, Alexandra Fusai, Chanda Rubin, Conchita Martínez, Anna Smashnova, Ruxandra Dragomir, Anabel Medina Garrigues and Flavia Pennetta.']], ['Monica Seles', ['Monica Seles ( ; Hungarian: \"Szeles Mónika\" , Serbian: Моника Селеш, \"Monika Seleš\" , ] , born December 2, 1973) is a former Yugoslav world no. 1 professional tennis player and a member of the International Tennis Hall of Fame.', ' An ethnic Hungarian, she was born and raised in Novi Sad, SFR Yugoslavia.', ' She became a naturalized American citizen in 1994 and also received Hungarian citizenship in June 2007.', ' She won nine Grand Slam singles titles, eight of them while representing Yugoslavia, and the final one while representing the United States of America.']], ['1997 Challenge Bell – Doubles', ['Debbie Graham and Brenda Schultz-McCarthy were the defending champions, but decided not to compete together.', ' Graham partnered with Mariaan de Swardt, but lost in the semifinals to Alexandra Fusai and Nathalie Tauziat.', ' Schultz-McCarthy partnered with Rebecca Jensen, but lost in the first round to Patricia Hy-Boulais and Chanda Rubin.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.635\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8013e45542995d8a8ddf41', 'answer': 'North Sea', 'question': 'Cowie Castle at Stonehaven, and Allardice Castle near Inverbervie, are part of a coastal chain of castles in Scotland overlooking which sea?', 'supporting_facts': [['Allardice Castle', 3], ['Cowie Castle', 0], ['Cowie Castle', 1]], 'context': [['Chapel of St Mary and St Nathalan', ['The Chapel of St. Mary and St. Nathalan is a ruined chapel overlooking the North Sea immediately north of Stonehaven, in the Mearns of Scotland, along the northern shoreline of Stonehaven Bay.', ' (Watt, 1985) The founding of this Christian place of worship is associated with St. Nathalan.', ' who lived circa 650 AD.', ' The structure is alternatively known as Cowie Chapel.']], ['Bervie Water', ['Bervie Water (Scottish Gaelic: \"Uisge Bhiorbhaigh\" ) is a river in Aberdeenshire, Scotland which rises in the Drumtochty Forest and flows across The Mearns to reach the North Sea at Inverbervie.', ' Approximately two kilometres upstream of the North Sea, the Bervie Water flows through the grounds of Allardice Castle.', ' The Bothenoth Burn (Burn of Healing) joins the Bervie Water to the east of Arbuthnott House.', ' At Arbuthnott the river forms a steep-sided valley where gardens were laid out on the south-facing slope.', ' It is a series of four terraces across which run diagonal intersecting grass paths.']], ['Allardice Castle', ['Allardice Castle (also spelled Allardyce) is a sixteenth-century manor house in Kincardineshire, Scotland.', ' This monument is occupied by the Cowie family and is approximately 1.5 kilometres northwest of the town of Inverbervie.', ' The Bervie Water flows around Allardice Castle on both sides.', ' Allardice may be viewed as one of a chain of coastal castles; to the north are Dunnottar Castle (ruined), Fetteresso Castle, Cowie Castle (ruined) and Muchalls Castle.', ' The castle is a category A listed building.']], ['Stonehaven Bay', ['Stonehaven Bay is a natural harbour in Aberdeenshire, Scotland.', ' The town of Stonehaven is built along the shore of Stonehaven Bay.', ' Nearby historical features include Fetteresso Castle, Stonehaven Tolbooth, Dunottar Castle and Muchalls Castle.']], ['Inverallochy Castle', ['Inverallochy Castle is a ruined courtyard castle near the village of Inverallochy in the Buchan area of north-east Scotland.', ' It lies 0.5 mi south of Cairnbulg Castle near Fraserburgh.', ' It was described by W. Douglas Simpson as one of the nine castles of the Knuckle, referring to the rocky headland of north-east Aberdeenshire.']], ['Cowie Castle', ['Cowie Castle is a ruined fortress in Aberdeenshire, Scotland.', ' The site lies at the northern end of Stonehaven on the North Sea coast.', ' To the immediate south is the Cowie Bridge crossing of the Cowie Water.', ' Evidence of prehistoric man exists in the vicinity dating to the Iron Age in the form of ring cairns.']], ['Cowie Water', ['The Cowie Water (Scottish Gaelic: \"Uisge Chollaidh\" ) is a river rising in the Grampian Mountains in Aberdeenshire, Scotland that discharges to the North Sea in the northern part of Stonehaven.', ' south of the ruined Cowie Castle.', ' Tributaries of the Cowie Water include the Burn of Monboys, which drains the area to the north, in which the archaeological site Raedykes Roman Camp is situated; and Cowton Burn.']], ['Mearns FM', ['Mearns FM is a community run radio station based in the North East of Scotland.', ' There are transmitters in Laurencekirk, Inverbervie and Stonehaven leading to a coverage area stretching from St Cyrus to Newtonhill.', ' The studio is located in Stonehaven Town Hall.']], ['Cowie Bridge', [\"Cowie Bridge is a roadway bridge across the Cowie Water in Stonehaven, Scotland near the river's mouth at the North Sea.\", ' This construction is a listed historical structure in Aberdeenshire.', ' Historically the area in the vicinity of the Cowie Bridge site has been an old fishing village known as Cowie Village.', ' Between the Cowie Bridge and the North Sea, a new pedestrian bridge is planned, which will also support a new pipeline structure.', ' The site of Cowie Bridge is approximately the point of the southern terminus of the Causey Mounth trackway, which was the only available medieval route crossing the coastal Grampian Mountains northerly by way of Muchalls Castle and Gillybrands.']], ['Fetteresso Castle', ['Fetteresso Castle is a 14th-century towerhouse, rebuilt in 1761 as a Scottish gothic style Palladian manor, with clear evidence of prehistoric use of the site.', ' It is situated immediately west of the town of Stonehaven in Kincardineshire slightly to the west of the A90 dual carriageway.', ' Other notable historic fortified houses or castles in this region are Dunnottar Castle, Muchalls Castle, Fiddes Castle, Cowie Castle and Monboddo House.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.636\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abb729b5542993f40c73af4', 'answer': \"The highway's eastern terminus\", 'question': 'On what part of U.S Route 56 can you listen to the KCTV?', 'supporting_facts': [['KCTV', 0], ['U.S. Route 56', 1]], 'context': [['Ohio State Route 29', ['State Route 29 (SR 29) is an east–west state highway in the west-central portion of the U.S. state of Ohio.', ' Its western terminus is at the Indiana state line near Celina, where State Road 67 continues west.', ' It continues east to St. Marys where it junctions with U.S. Route 33.', ' In that town, it also crosses State Route 66, State Route 116, and State Route 703, which was its former alignment before a divided highway was built.', ' After turning south it crosses State Route 219 in New Knoxville and then has an interchange with Interstate 75, continuing into Sidney where it meets State Route 47.', ' Still going southeast, it briefly joins State Route 235 before turning east and then south again to enter Urbana.', ' Here the route joins U.S. Route 36, and the concurrency intersects with U.S. Route 68 and State Route 54.', ' From there, State Route 29 leaves U.S. Route 36 and continues to Mutual, intersecting with State Route 161, and State Route 56 shortly after; later, in Mechanicsburg, the route intersects with State Route 4.', ' The route then intersects with State Route 38, U.S. Route 42, and Interstate 70 before reaching its eastern terminus at U.S. Route 40 on the western edge of West Jefferson.']], ['SEPTA Route 56', [\"SEPTA's Trolley Route 56, the Erie and Torresdale Avenues Line is a streetcar line that is now served by bus.\", ' It is operated by the Southeastern Pennsylvania Transportation Authority (SEPTA) in Philadelphia, Pennsylvania, United States.', ' The line runs between the Tioga and Tacony neighborhoods primarily along Erie Avenue and Torresdale Avenue.', ' Route 56 was one of three \"suspended\" by the SEPTA board effective June 12, 1992.', ' The two others, Routes 15 and 23, were then also operated by buses as of the same year; however, the Route 15 Trolley has since been restored back to trolley as of September 5, 2005.']], ['Maryland Route 56', ['Maryland Route 56 (MD 56) is a state highway in the U.S. state of Maryland.', ' Known as Big Pool Road, the state highway runs 8.08 mi from Interstate 70 (I-70) in Big Pool east to MD 68 near Pinesburg in western Washington County.', ' MD 56, which features a pair of one-lane bridges, provides access to Fort Frederick State Park and the eastern trailhead of the Western Maryland Rail Trail.', ' The state highway was first constructed in the mid 1910s from what is now U.S. Route 40 (US 40) south to Big Pool.', ' The highway was extended east toward the village of Big Spring in the early 1930s and to MD 68 in the late 1950s.', ' MD 56 was truncated at I-70 in the mid-1990s.']], ['New York State Route 56', ['New York State Route\\xa056 (NY\\xa056) is a north–south state highway in eastern St. Lawrence County, New York, in the United States.', ' The southern terminus of the route is at an intersection with NY\\xa03 in the town of Colton.', ' Its northern terminus is at a junction with NY\\xa037 southwest of Massena in the town of Louisville.', ' NY\\xa056 follows the Raquette River for most of its length and serves the village of Potsdam, where it passes by the campus of Clarkson University.']], ['Virginia State Route 56', ['State Route 56 (SR 56) is a primary state highway in the U.S. state of Virginia.', ' The state highway runs 60.87 mi from U.S. Route 11 (US 11) at Steeles Tavern east to US 60 near Buckingham.', ' SR 56 is the main east–west highway of Nelson County.', ' The state highway connects the county seat of Lovingston with Buckingham to the east and the Blue Ridge Mountains and the Shenandoah Valley to the west.']], ['New Jersey Route 77', ['Route 77 is a state highway in the southern part of the U.S. state of New Jersey.', ' It runs 22.55 mi from an intersection with Route 49 in Bridgeton, Cumberland County north to a junction with Route 45 in Mullica Hill, Gloucester County.', ' It is a mostly two-lane, undivided road traversing through farmland in Cumberland, Salem, and Gloucester Counties.', ' Along the way, Route 77 intersects Route 56 in Upper Deerfield Township and U.S. Route 40 (US 40) in Upper Pittsgrove Township.']], ['New Jersey Route 56', ['Route 56 is a state highway in the southern part of New Jersey, United States.', ' Also known as Landis Avenue, it runs 9.19 mi from an intersection with Route 77 and County Route 622 in Upper Deerfield Township, Cumberland County, to an intersection with Route 47 (Delsea Drive) in Vineland, Cumberland County.', ' The route serves as a connector between Bridgeton and Vineland.', ' West of the interchange with Route 55 in Vineland, Route 56 is a two-lane undivided road that passes through rural areas of Cumberland County, also entering a corner of Salem County.', ' East of Route 55, the route is a four-lane locally maintained road that runs through commercial areas of Vineland.']], ['U.S. Route 56', ['U.S. Route 56 (US 56) is an east–west United States highway that runs for 640 mi in the Midwestern United States.', \" The highway's eastern terminus is at U.S. Route 71 in Kansas City, Missouri.\", ' Its western terminus is at Interstate 25 Business in Springer, New Mexico.', ' Much of it follows the Santa Fe Trail.']], ['Illinois Route 56', ['Illinois Route 56 (IL 56) is an east–west state road in northern and northeastern Illinois.', ' It runs from the interchange of Illinois Route 47 at U.S. Route 30 (US 30) in Sugar Grove east to US 12/US 20/US 45 (Mannheim Road) by Bellwood.', ' This is a distance of 32.52 mi .']], ['KCTV', ['KCTV, virtual channel 5 (UHF digital channel 24), is a CBS-affiliated television station licensed to Kansas City, Missouri, United States and also serving Kansas City, Kansas.', ' The station is owned by the Meredith Local Media subsidiary of the Meredith Corporation, as part of a duopoly with MyNetworkTV affiliate KSMO-TV (channel 62).', ' The two stations share studio facilities located on Shawnee Mission Parkway (U.S. 56/U.S. 169) in Fairway, Kansas; KCTV maintains transmitter facilities located on East 31st Street in the Union Hill section of Kansas City, Missouri (adjacent to the studios of PBS member station KCPT (channel 19)).', ' On cable, KCTV is available on Charter Spectrum, Comcast Xfinity and Consolidated Communications channel 3, and Google Fiber and AT&T U-verse channel 5.', ' There is a high definition feed provided on Spectrum digital channel 1209, Xfinity channel 803, Consolidated channel 620 and U-verse channel 1005.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.636\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adf5c175542995ec70e8fd0', 'answer': 'Jaguar Land Rover', 'question': 'What company headquartered in Whitley, Coventry, United Kingdom will start production in Nitra in 2018?', 'supporting_facts': [['Automotive industry in Slovakia', 3], ['Jaguar Land Rover', 0]], 'context': [['Avro Canada C102 Jetliner', ['The Avro C102 Jetliner was a Canadian prototype medium-range turbojet-powered jet airliner built by Avro Canada in 1949.', ' It was beaten to the air by only 13 days by the de Havilland Comet, thereby becoming the second jet airliner in the world.', ' The name \"Jetliner\" was chosen as a shortening of the term \"jet airliner\", a term which is still in popular usage in Canada and the United States.', ' The aircraft was considered suitable for busy routes along the US eastern seaboard and garnered intense interest, notably from Howard Hughes who even offered to start production under license.', \" However continued delays in Avro's all-weather interceptor project, the Avro CF-100, led to an order to stop working on the project in 1951, with the prototype Jetliner later cut up for scrap.\"]], ['GMT T1XX', ['The GMT T1XX is the assembly code for an vehicle platform architecture in development by General Motors for its line of full-size trucks and large SUVs that has been announced to start production in the fall of 2018 for the 2019 model year.', ' The \"XX\" is a place holder for the last two digits of the specific assembly code for each model.', ' As an example, the project code for the Suburban is T1YC.', ' The platform is intended to replaced the GMT K2XX series that was introduced in April 2013 for the trucks, followed by the December 2013 production of large SUVs.']], ['Jaguar Land Rover', ['Jaguar Land Rover Automotive PLC is the holding company of Jaguar Land Rover Limited, a British multinational automotive company with its headquarters in Whitley, Coventry, United Kingdom, and a subsidiary of Indian automaker Tata Motors.', ' The principal activity of Jaguar Land Rover Limited is the design, development, manufacture and sale of vehicles bearing the Jaguar and Land Rover (including Range Rover) marques.', ' Both marques have long and complex histories prior to their merger, going back to the 1940s, first coming together in 1968 as part of the ill-fated British Leyland conglomerate; and later existed independently of each other as subsidiaries of BMW (in the case of Land Rover), and Ford Motor Company (in the case of Jaguar); Ford later acquired Land Rover from BMW in 2000 following the break-up of the former Rover Group; which was effectively the remainder of British Leyland.']], ['Coventry City Derby Dolls', ['Coventry City Derby Dolls (CCDD) are Coventry’s first and only all female flat track roller derby league based in Coventry, United Kingdom.']], ['Fulton ethanol plant', [\"Sunoco's Fulton ethanol plant in Fulton, New York is the first such facility owned by the company.\", ' The plant is spread over an area of 115 acres in Riverview Business Park and includes a 250,000\\xa0ft² brewhouse.', ' The plant has the capacity to produce 85m gallons of ethanol annually.', ' Northeast Biofuels opened the plant in 2008; however, design flaws led to growth of bacteria in pipes that were difficult to clean.', ' Northeast Biofuels tried to rectify the problem and fix the pipes but failed.', ' The company filed for bankruptcy in 2009.', ' Sunoco bought the plant from Northeast Biofuels in June 2009 for $8.5m.', ' The company spent $25m, and contracted ICM, to repair the design flaws and start production.', ' The refurbished facility became fully operational with the production of the first batch of ethanol in June 2010.']], ['Bablake School', ['Bablake School is a co-educational Independent school located in Coventry, England and founded in 1344 by Queen Isabella, making it one of the oldest schools in the United Kingdom (List of the oldest schools in the United Kingdom).', ' Bablake is part of the Coventry School Foundation, a registered charity, along with King Henry VIII School, King Henry VIII Preparatory School and Cheshunt School.', ' The current headmaster is John Watson, who succeeded Dr Stuart Nuttall following his retirement in 2006.', ' Today Bablake is a selective, fee-paying independent school and a member of the HMC.']], ['Automotive industry in Slovakia', [\"Since 2007, Slovakia has been the world's largest producer of cars per capita, with a total of 1,040,000 cars manufactured in 2016 alone in a country with 5 million people.\", ' With production of more than million cars in 2016, Slovakia was 20th in the list of worldwide car production by country and the 7th largest car producer in the European Union.', \" Automotive is the largest industry in Slovakia with a share of 12% on the Slovak GDP in 2013 which was 41% of industrial production and 26% of Slovakia's export.\", ' 80.000 people were employed in the automotive industry in 2014, which will be even increased when Jaguar Land Rover starts production in Nitra in 2018.']], ['The London Taxi Company', ['London Electric Vehicle Company Ltd., formerly The London Taxi Corporation Ltd trading as The London Taxi Company; formerly part of Manganese Bronze Holdings plc is an automotive engineering company headquartered in Coventry, United Kingdom, and a wholly owned subsidiary of Chinese automaker Geely.', ' It was founded in 1899 and its principal activity is the design, development and production of taxicabs. With the launch of the new TX electric taxi in 2017, the company changed its name to London Electric Vehicle Company Ltd. and announced its intentions to begin production of electric commercial vehicles in addition to the taxi cabs.']], ['Lockheed L-100 Hercules', ['The Lockheed L-100 Hercules is the civilian variant of the prolific C-130 Hercules military transport aircraft made by the Lockheed Corporation.', ' Its first flight occurred in 1964.', ' Longer L-100-20 and L-100-30 versions were developed.', ' L-100 production ended in 1992 with 114 aircraft delivered.', ' An updated variant of the model, LM-100J, have completed its first flight in Marietta, Georgia on May 25, 2017, and is set to start production in 2018-2019.']], ['Coventry University Business School', ['Coventry Business School is a business school located in Coventry, United Kingdom.', ' It is a department of Coventry University and its Faculty of Business and Law.', ' The School offers a number of undergraduate and postgraduate degrees in subjects such as economics, marketing, event management and applied management.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.637\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adda8d45542995b365fab59', 'answer': 'Carol Ann Duffy', 'question': 'Which Scottish playwright wrote the poem \"Havisham\"?', 'supporting_facts': [['Havisham', 0], ['Carol Ann Duffy', 0]], 'context': [['Havisham', ['\"Havisham\" is a poem written in 1993 by Carol Ann Duffy.', ' It responds to Charles Dickens\\' character \"Miss Havisham\" from his novel \"Great Expectations\", looking at Havisham\\'s mental and physical state many decades after being left standing at the altar, when the bride-to-be is in her old age.', \" It expresses Havisham's anger at her fiancé and her bitter rage over wedding-day trauma and jilted abandonment.\", \" Duffy's use of language is very powerful and passionate.\", ' Throughout the poem oxymorons and juxtaposition such as \"\"Beloved sweetheart bastard\"\" and \"\"Love\\'s hate\"\" portrays the ambivalence and restless uncertainty of the character, while a sexual fantasy reveals both the unrequited love and the passion that remains within Havisham following the wedding, a devastation from which her heart has never recovered.']], ['Cecil Taylor (playwright)', ['Cecil Philip Taylor (1929 – 1981), usually credited as C.P. Taylor, was a Scottish playwright.', ' He wrote almost 80 plays during his 16 years as a professional playwright, including several for radio and television.', ' He also made a number of documentary programmes for the BBC.', ' His plays tended to draw on his Jewish background and his Socialist viewpoint, and to be written in dialect.']], ['W. Gordon Smith', ['W. Gordon Smith (13 December 1928 – 13 August 1996) was a Scottish playwright.', ' He was born in Edinburgh and lived most of his life there.', ' He wrote many plays including the one man show, \"Jock\", made famous by Russell Hunter.', ' He also wrote the lyrics, \"Come By The Hills\", set to the tune of the traditional Irish song, \"Buachaill o\\'n Éirne Mé\".', ' In addition he wrote books on the artist, William George Gillies, and on the author, Robert Louis Stevenson and was instrumental in bringing the Scottish arts scene to BBC Scotland.']], ['Andrew Dallmeyer', ['Andrew Dallmeyer (10 January 1945 - 21 May 2017) was a Scottish playwright, theatre director and actor.', ' He wrote over 75 plays, including the \"Opium Eater\" and directed more than 50 productions.', ' His plays have won a number of awards, including a Scottish BAFTA, and they have been played on BBC Radio.']], ['Glenn Chandler', ['Glenn Chandler (born 12 March 1949) is an award-winning Scottish playwright and novelist.', ' He has written plays for theatre and radio, original screenplays for television and films, television series, and novels.', ' His best-known work is the Scottish television detective series \"Taggart\", which is broadcast around the world.']], ['Carol Ann Duffy', ['Dame Carol Ann Duffy (born 23 December 1955) is a Scottish poet and playwright.', \" She is Professor of Contemporary Poetry at Manchester Metropolitan University, and was appointed Britain's Poet Laureate in May 2009.\", ' She is the first woman, the first Scot, and the first openly LGBT person to hold the position.']], ['D C Jackson', ['Daniel Craig Jackson is a Scottish playwright, born in 1980.', ' His first full-length play \"The Wall\" premiered at the Tron Theatre in Glasgow in 2008.', \" It was produced by Borderline Theatre Company and was nominated for several awards including the Best New Play at the Critics' Awards for Theatre in Scotland and the Saltire Society Scottish First Book of the year.\", ' The sequel \"The Ducky\" was also produced by Borderline Theatre Company and toured in 2009.', ' In 2010 he finished his \"Stewarton Trilogy\" with \"The Chooky Brae\".', \" His play My Romantic History' (which starred Iain Robertson) won a Scotsman Fringe First at the 2010 Edinburgh Festival and sold out its run at the Bush Theatre London.\", ' He also took part in the Bush Theatre\\'s 2011 project \"Sixty Six Books\" where he contributed a piece based upon a book of the King James Bible.', \" In 2012 Jackson's play The Marriage of Figaro, an adaptation of the stage comedy by Beaumarchais and later opera by Wolfgang Amadeus Mozart was premiered at the Royal Lyceum Theatre, Edinburgh.\", \" In 2013 Jackson's play Threeway premiered at the Edinburgh Festival Fringe, Edinburgh.\", \" In 2014, another of Jackson's work Kill Johnny Glendenning received its premiere at the Lyceum before transferring to Glasgow's Citizens Theatre.\"]], ['Margaret Turnbull (screenwriter)', ['Margaret Turnbull (17 November 1872 – 12 June 1942) was a Scottish playwright and screenwriter.', ' She wrote for 51 films between 1914 and 1939.', ' She also wrote novels, such as \"The Close Up\" (1918), \"Alabaster Lamps\" (1925) and \"The Bride\\'s Mirror\" (1934).', ' She was born in Glasgow, Scotland and died in Yarmouthport, Massachusetts.', ' She was the sister of producer Hector Turnbull.', ' She worked for the Famous Players-Lasky studios in Islington, England, and also spent some of her career in Hollywood.']], ['Paul Morris (writer)', ['Paul Morris (born 1958) is a Scottish playwright and screenwriter, born in Uddingston, South Lanarkshire.', ' His early original plays in Scottish theatre include \"Three Wee Kings\", \"Stef, Crabs and Wilsy\" and \"Transformer\", a reworking of Franz Kafka\\'s \"The Metamorphosis\".', ' More recently, he has co-written and directed a low-budget comedy feature \"Siamese Cop\" (\"Two Cops One Jacket\"), co-written an animated feature \"Duck Ugly\", produced by Millimages and written an original screenplay \"Partiendo Atomos\", set in Buenos Aires and directed by John Dickinson.', ' In 2011 his first novel, \"Pa Weathery\\'s Chickens\", was published by \"Night Publishing\".']], ['Ann Marie Di Mambro', ['Ann Marie Di Mambro (born 18 June 1950) is a Scottish playwright and television screenwriter of Italian extraction.', \" Her theatre plays have been performed widely; they are also published individually and in collections and are studied in schools for the Scottish curriculum's Higher Drama and English.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.638\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab6c65e554299710c8d1f58', 'answer': '8,515', 'question': 'What was the population of the town as of 2000 where the historic tugboat Catawissa is located?', 'supporting_facts': [['Catawissa (tugboat)', 0], ['Waterford, New York', 0], ['Waterford, New York', 1]], 'context': [['Ancón, Panama', ['Ancón is a corregimiento in Panamá District, Panamá Province, Panama with a population of 29,761 as of 2010.', ' Its population as of 1990 was 11,518; its population as of 2000 was 11,169.', ' It is sometimes considered a suburb or small town within Panama City, northeast of the limits of the town of Balboa.', ' Ancon Hill is also the name of a large hill that overlooks Panama City and once served as a form of protection from pirates and sea invasion.', ' The township was originally located around this hill, and was created to house employees of the Panama Canal during its construction.', ' As part of the construction effort, the historic Gorgas Army Hospital was founded and built on the hillside.', ' The first ship to officially transit the canal, SS\\xa0\"Ancon\" , was named after the district.', ' The community continued to serve as housing for employees of the Panama Canal Company until 1980, when parts of it began to be turned over to the Panamanian government under the 1977 Torrijos-Carter Treaties.', ' Modern-day Ancón is a \"corregimiento\" (the Panamanian equivalent of a suburb in the United States) of Panama City, serving mainly as a residential area.', ' The Gorgas Army Hospital building is now the Panamanian Oncology Hospital, primarily used for cancer research.', \" The area also houses Panama's Supreme Court, just a few feet away from the Gorgas Army Hospital building, and several Smithsonian Tropical Research Institute buildings for research into tropical biology.\", ' Ancón is also a parish (\"parroquia\") of the District of Panama, located in the Panama Canal adjacent area.']], ['Southern Columbia Area School District', ['Southern Columbia Area School District is a small, rural, public school district located in Catawissa, Pennsylvania.', ' It serves communities in two counties.', ' In Columbia County the district serves: Catawissa Borough, Catawissa Township, Locust Township, Roaring Creek Township, and Cleveland Township.', ' This includes the boroughs of: Slabtown and Numidia.', ' In Northumberland County it serves Ralpho Township.', ' Southern Columbia Area School District encompasses approximately 108 sqmi .', ' According to 2000 federal census data, it serves a resident population of 9,803.', \" By 2010, the US Census Bureau reported that the district's population increased to 10,386 people.\", ' The per capita income of residents was $18,969 in 2009, while the median family income was $45,889.', ' In the Commonwealth, the median family income was $49,501 and the United States median family income was $49,445, in 2010.', ' The district is one of the 500 public school districts of Pennsylvania.']], ['K. Whittelsey (tugboat)', ['K. Whittelsey is a historic tugboat located at Kingston, Ulster County, New York.', ' She was built in 1930, and is a 185 gross ton diesel tugboat measuring 90 feet, 6 inches, long.', ' She was built by Spedden Shipbuilders of Baltimore, Maryland and towed oil barges.']], ['Catawissa (tugboat)', ['Catawissa was a historic tugboat located at Waterford in Saratoga County, New York.', ' She was built in 1896-1897 by Harlan and Hollingsworth of Wilmington, Delaware for the Philadelphia and Reading Railroad to tow coal barges between ports on the Eastern Seaboard.', ' She was 158 feet in length, 19 feet in beam and 18 feet in depth.', ' She was registered at 558 gross tons.', ' She had a riveted steel framed and plated hull.']], ['Warrensburg, New York', ['Warrensburg is a town in Warren County, New York, United States.', ' It is centrally located in the county, west of Lake George.', ' It is part of the Glens Falls Metropolitan Statistical Area.', ' The town population was 4,255 at the 2000 census.', ' While the county is named after General Joseph Warren, the town is named after James Warren, a prominent early settler.', ' U.S. Route 9 passes through the town, which is immediately west of Interstate 87 (The Northway).', \" According to the 2000 United States Census, the town's main hamlet, also recognized by the U.S. Census Bureau as a census-designated place (CDP), comprises less than one-fifth of the town's total area, yet has about 75% of the town's population.\", \" The Warrensburg CDP's population density is more than fourteen times that of the town outside the CDP.\"]], ['Jeff Davis County Courthouse (Texas)', ['The Jeff Davis County Courthouse is located in the town of Fort Davis, the seat of Jeff Davis County in the U.S. state of Texas.', ' The courthouse was constructed between 1910-1911 and added to the National Register of Historic Places in 2002.', ' The Texas Historical Commission (THC) has also designated the building as a Recorded Texas Historic Landmark since 2000 and, along with the surrounding courthouse square, as a State Antiquities Landmark since 2003.', ' The surrounding county and county seat, along with the nearby historic frontier fort at Fort Davis National Historic Site, are named after Jefferson Davis, who served as U.S. war secretary at the time of the establishment of the fort and the town, and who would later become president of the Confederate States of America during the Civil War.']], ['Taifa, Accra', ['Taifa is a town in the Ga East Municipal District, a district in the Greater Accra Region of south-eastern Ghana near the capital Accra.', ' Taifa is the twenty-sixth largest settlement in Ghana, in terms of population, with a population of 68,459 people.', ' Taifa is located in the northwest suburbs area of Accra.', ' It has a breakpoint on a railway line and a small park located on the northern edge of the location of the Taifa Ghana Atomic Energy Commission.', ' At the Ghana 2000 census of 26 March 2000, the population was 26,145 inhabitants living in the city.', ' Projections of 1 January 2007 estimated the population to be 48,927 inhabitants.', ' In the census of 1984 there was only 1,009 inhabitants.', ' The strong population growth of the Town is influenced by, among other things, a large number of illegal immigrants from west African countries who move to towns and villages near the industrial town of Tema, just to find a job.']], ['Waterford, New York', ['Waterford is a town in Saratoga County, New York, United States.', ' The population was 8,515 at the 2000 census.', ' The name of the town is derived from its principal village, also called Waterford.', ' The town and village are in the southeast corner of Saratoga County, and north-northwest of Troy, New York.', ' It is located at the junction of the Erie Canal and the Hudson River.']], ['Mogollon, New Mexico', ['Mogollon, also called the Mogollon Historic District, is a former mining town located in the Mogollon Mountains in Catron County, New Mexico, in the United States.', ' Located east of Glenwood and Alma, it was founded in the 1880s at the bottom of Silver Creek Canyon to support the gold and silver mines in the surrounding mountains.', ' A mine called \"Little Fannie\" became the most important source of employment for the town\\'s populus.', ' During the 1890s Mogollon had a transient population of between 3,000 and 6,000 miners and, because of its isolation, had a reputation as one of the wildest mining towns in the West.', ' Today Mogollon is listed as Fannie Hill Mill and Company Town Historic District on the National Register of Historic Places.']], ['East Grinstead', ['East Grinstead is a town and civil parish in the northeastern corner of Mid Sussex district of West Sussex in England near the East Sussex, Surrey, and Kent borders.', ' It lies 27 mi south of London, 21 mi north northeast of Brighton, and 38 mi east northeast of the county town of Chichester.', ' The civil parish covers an area of 2443.45 ha and had a population of 23,942 persons in the 2001 census.', ' The population of the town at the 2011 Census was 26,383.', ' Nearby towns include Crawley and Horley to the west, Tunbridge Wells to the east and Redhill and Reigate to the northwest.', ' The town is contiguous with the village of Felbridge to the northwest.', ' Until 1974 East Grinstead was the centre for local government - East Grinstead Urban District Council - and was located in the county of East Sussex.', ' East Grinstead, along with Haywards Heath and Burgess Hill, as part of the former Cuckfield Rural District Council, came together as Mid-Sussex; moving to the jurisdiction of West Sussex County Council.', ' The town has many historic buildings and is located on the Greenwich Meridian.', ' It is located in the Weald and Ashdown Forest lies to the south-east of the town.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.638\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae40d405542996836b02c28', 'answer': 'North Avenue and First Avenue', 'question': 'At what intersection was the former home of the wooden roller coaster now located at Six Flags Great America in Gurnee, Illinois located?', 'supporting_facts': [['Little Dipper (Six Flags Great America)', 0], ['Little Dipper (Six Flags Great America)', 1], ['Kiddieland Amusement Park', 0]], 'context': [['Viper (Six Flags Magic Mountain)', ['Viper is a steel roller coaster made by Arrow Dynamics of the United States.', ' The roller coaster is located in the Baja Ridge area of Six Flags Magic Mountain in Valencia, California.', ' Viper is the last of the three 7-looper roller coasters built by Arrow Dynamics to remain operating.', ' The other two, Shockwave at Six Flags Great America and the Great American Scream Machine at Six Flags Great Adventure, were demolished in 2002 and 2010, respectively.', ' The roller coaster replaced a HUSS ride type named Condor.']], ['Key Lime Cove', ['KeyLime Cove was a resort and indoor waterpark located in Gurnee, Illinois near Six Flags Great America.', ' KeyLime Cove was listed as the \"Official Resort of Six Flags Great America\".', ' The resort was located about an hour away from Chicago and less than an hour from Milwaukee.', ' The resort closed on April 19, 2017 and will be under renovations for the next year to transform the tropical themed resort to a north woods themed look.', ' Great Wolf Resorts, a Madison-based company with over 14 resorts opened and 4 under construction/renovation, acquired the property on February 6 at the Gurnee Village board meeting the night before.', ' The indoor water park will also be expanded by about 20,000 square feet.', ' In addition, further facilities are set to include an outdoor water park, more shopping and other amenities.', ' The company’s Head of Development, Bryson Heezen, reportedly told Gurnee village board that fresh family rooms with themes such as a “wolf den,” new restaurants, mini-bowling and ropes courses were also on the cards.', ' Guests that booked a future stay after April 20, 2017 will receive a notification that your stay was cancelled and a refund for any deposit held is being processed.', ' Due to the large number of refunds being processed, it may take up to 30 days for guests to receive a full refund.', ' Guests holding a KeyLime Cove gift card and are eligible to receive reimbursement for any remaining balance on the gift card.', ' Refunds of gift card balances are currently being processed and will be mailed to guests within 30 days.']], ['Apocalypse (Six Flags America)', [\"Apocalypse is a steel stand-up roller coaster located at Six Flags America in Prince George's County, Maryland.\", ' The ride made its debut in 1990 as Iron Wolf at Six Flags Great America before being relocated to Six Flags America and renamed to Apocalypse.', ' The roller coaster was the first built by Swiss manufacturer Bolliger & Mabillard.', ' When known as Iron Wolf, the roller coaster held the records of the highest (100 ft ) and fastest (55 mph ) stand-up looping roller coaster in the world before losing them to other roller coasters in 1992 and 1996.']], ['Little Dipper (Six Flags Great America)', ['Little Dipper is a wooden roller coaster located at Six Flags Great America in Gurnee, Illinois.', ' It was originally built in 1950 by Philadelphia Toboggan Coasters and previously operated at Kiddieland Amusement Park in Melrose Park, Illinois until the park closed in September 2009.', ' On November 24, 2009, Six Flags Great America purchased the ride for $33,000 at an auction.', ' The ride was relocated and reopened on May 27, 2010, in its original configuration within the Yukon Territory section of Six Flags Great America.']], ['Thunderbolt (Six Flags New England)', ['Thunderbolt is a wooden roller coaster located at Six Flags New England.', ' Opened in 1941, It was designed by Harry Baker and Harry Traver, and built by Joseph Drambour.', ' Thunderbolt is the oldest roller coaster at Six Flags New England.', ' It is also the oldest roller coaster in any Six Flags park (the Wild One at Six Flags America was built in 1917, but it was relocated from Paragon Park and has only been at Six Flags America since 1986).', ' The single PTC train has 4 cars, and an individual lap bar and seatbelt for each person.', \" An attendant has to manually unlock each car's lap bars by stepping on and pushing down a release bar at the front of each car.\", ' Thunderbolt was dedicated an ACE Coaster Landmark on August 2, 2008.']], ['American Eagle (roller coaster)', ['American Eagle is a wooden racing roller coaster located at Six Flags Great America.', ' It was the first wooden roller coaster designed by Intamin of Switzerland and was built in 1981 by the contracting firm Figley-Wright.', ' While the records have since been broken, American Eagle had the longest drop and fastest speed among wooden roller coasters when it debuted and is still recognized as a top racing coaster in the United States.', ' In 2006, American Eagle celebrated its Silver Anniversary (25th Anniversary).']], ['Goliath (Six Flags Great America)', ['Goliath is a wooden roller coaster at Six Flags Great America in Gurnee, Illinois.', ' Manufactured by Rocky Mountain Construction and designed by Alan Schilke, the ride set three world records for wooden roller coasters when it opened to the public on June 19, 2014: the longest drop at 180 ft , the steepest drop at 85°, and the fastest speed at 72 mph .', ' The 100-second ride also features two inversions.']], ['Viper (Six Flags Great America)', ['Viper is a wooden roller coaster located at Six Flags Great America in Gurnee, Illinois, which opened in 1995.', ' Viper features a layout that is a mirror image of the Coney Island Cyclone and is the only roller coaster ever to be built directly by Six Flags.', ' Viper is also the only wooden roller coaster that has ever carried this name, (all the other Viper roller coasters, both operating and not, have been steel roller coasters).', ' It was built by Rygiel Construction.']], ['El Toro (Six Flags Great Adventure)', ['El Toro, a Spanish term meaning The Bull, is a wooden roller coaster at Six Flags Great Adventure in Jackson, New Jersey.', ' Designed by Intamin of Switzerland, it opened to the public on June 11, 2006.', ' Intamin also worked with members of Rocky Mountain Construction to build the ride.', ' When it opened, it had the steepest drop of any wooden roller coaster in the world at 76 degrees, until the record was broken by T Express in 2008 by one degree.', ' Overall, its structure height of 181 ft is ranked fourth, its drop height of 176 ft is ranked second, and its top speed of 70 mph is ranked fourth among all wooden roller coasters in the world.', ' It was also the first wooden roller coaster to use a cable lift as opposed to the traditional chain lift.']], ['Kiddieland Amusement Park', ['Kiddieland Amusement Park was an amusement park located just west of Chicago at the corner of North Avenue and First Avenue in Melrose Park, Illinois.', ' It was home to several classic rides including the \"Little Dipper\" roller coaster, which opened in 1950.', ' The park closed on September 27, 2009, and was demolished in 2010 to make way for a new Costco store.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.639\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac21bac5542991316484bf2', 'answer': 'Excalibur Hotel and Casino', 'question': 'What casino in Las Vegas which is now operated by MGM Resorts International was developed by William G. Bennett?', 'supporting_facts': [['William G. Bennett (gaming executive)', 3], ['Excalibur Hotel and Casino', 0], ['Excalibur Hotel and Casino', 1]], 'context': [['Monte Carlo Resort and Casino', ['The Monte Carlo Resort and Casino is a megaresort hotel and casino on the Las Vegas Strip in Paradise, Nevada, United States.', ' The hotel, with a height of 360 ft , has 32 floors, featuring a 102000 sqft casino floor with 1,400 slot machines, 60 table games, and 15 poker tables.', ' It is owned and operated by MGM Resorts International.', ' The hotel offers 2,992 guest rooms, including 259 luxury suites.', ' It is being converted from late 2016 to 2018 into the Park MGM, with the upper floors converted into a boutique hotel, NoMad Las Vegas.']], ['Slots-A-Fun Casino', ['Slots-A-Fun Casino is a casino on the Las Vegas Strip.', ' It is owned and operated by MGM Resorts International.', ' It is adjacent to Circus Circus Las Vegas.']], ['New York-New York Hotel and Casino', ['New York-New York Hotel & Casino is a hotel and casino located on the Las Vegas Strip at 3790 Las Vegas Boulevard South, in Paradise, Nevada.', ' It is owned and operated by MGM Resorts International.']], ['William G. Bennett (gaming executive)', ['William G. Bennett (November 16, 1924-December 22, 2002) was an American gaming executive and real estate developer.', ' Noted for pioneering Las Vegas as a destination for middle-class tourists and their families, he is best remembered for his establishment of gaming giant Circus Circus Enterprises in 1974.', ' He served as chairman of Circus Circus between 1974 and 1994.', ' Under his leadership Circus Circus would go on to develop numerous additional properties throughout Nevada, including the Excalibur and Luxor casinos in Las Vegas.', ' Following his departure from Circus Circus, Bennett purchased the Sahara Hotel and Casino in Las Vegas in 1995 and operated it until his death in 2002.']], ['Steve Wynn', ['Stephen Alan Wynn (\"né\" Weinberg; born January 27, 1942) is an American real estate businessman and art collector.', ' He is known for his involvement in the American luxury casino and hotel industry.', ' Early in his career he oversaw the construction and operation of several notable Las Vegas and Atlantic City hotels, including the Golden Nugget, the Golden Nugget Atlantic City, The Mirage, Treasure Island, the Bellagio, and Beau Rivage in Mississippi, and he played a pivotal role in the resurgence and expansion of the Las Vegas Strip in the 1990s.', ' In 2000, Wynn sold his company Mirage Resorts to MGM Grand Inc., resulting in the formation of MGM Mirage (now MGM Resorts International).', \" Wynn afterwards took his company Wynn Resorts public in an initial public offering, and he remains Wynn Resorts' CEO and Chairman of the Board.\", ' He is a member of the Republican Party.', ' Wynn is the finance chair of the Republican National Committee (RNC) since 2017.']], ['MGM Resorts International', ['MGM Resorts International is a global hospitality and entertainment company operating destination resorts in Las Vegas, Mississippi, New Jersey and Detroit, including Bellagio, MGM Grand, Mandalay Bay and The Mirage.', ' The company recently opened MGM National Harbor in Maryland and is developing MGM Springfield in Massachusetts.', ' It has a majority interest in MGM China Holdings Limited, which owns the MGM Macau resort and casino and is developing a gaming resort in Cotai.', ' MGM Resorts owns 50 percent of CityCenter in Las Vegas, which features ARIA Resort & Casino.', ' It has a majority controlling interest in MGM Growth Properties, a real estate investment trust.']], ['Delano Las Vegas', ['Delano Las Vegas, (formerly known as THEhotel), is a 45-story 1,117 room luxury suite hotel.', ' It is owned and operated by MGM Resorts International.', ' It is located within the Mandalay Bay complex on the Las Vegas Strip in Paradise, Nevada.', ' It was renovated and rebranded as the Delano Las Vegas on September 2, 2014, under a partnership between MGM and Morgans Hotel Group.']], ['Corey I. Sanders', ['Corey Sanders has served as Chief Operating Officer of MGM Resorts International since June 2010.', ' He oversees operations at the Company’s wholly owned properties, which in Nevada include Bellagio (resort), MGM Grand Las Vegas, Mandalay Bay, The Mirage, New York-New York Hotel and Casino, Monte Carlo Resort and Casino, Luxor Las Vegas, Excalibur Hotel and Casino, Circus Circus Las Vegas, Circus Circus Reno, Gold Strike Jean and Railroad Pass Casino.', ' He also oversees Beau Rivage (Mississippi) in Biloxi and Gold Strike Tunica, both in Mississippi, as well as MGM Grand Detroit.']], ['Excalibur Hotel and Casino', ['Excalibur Hotel and Casino is a hotel and casino located on the Las Vegas Strip in Paradise, Nevada, in the United States.', ' It is owned and operated by MGM Resorts International.']], ['CityCenter', ['CityCenter (also known as CityCenter Las Vegas) is a 16797000 sqft mixed-use, urban complex on 76 acre located on the Las Vegas Strip in Paradise, Nevada.', \" The project was started by MGM Resorts International; Dubai World became a joint partner during the project's construction phase.\", ' It is the largest privately funded construction project in the history of the United States.', ' The project is connected by a people mover system to adjacent MGM properties Monte Carlo Las Vegas and Bellagio Las Vegas.', ' As of 2015, the \"CityCenter\" branding has been largely retired, with the focus instead on the Aria brand of the development\\'s centerpiece property in names such as the \"Aria Express\" (formerly \"CityCenter Tram\") and \"Aria Art Collection\" (formerly \"CityCenter Art Collection\").']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.641\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adfffe755429925eb1afbd1', 'answer': 'omnisexuality', 'question': 'What was Mary Edna González sexuality? ', 'supporting_facts': [['Mary González', 0], ['Pansexuality', 0]], 'context': [['Mary Marcy', ['Mary Edna Tobias Marcy (May 8, 1877 – December 8, 1922) was an American socialist author, pamphleteer, poet, and magazine editor.', ' She is best remembered for her muckraking series of magazine articles on the meat industry, \"Letters of a Pork Packer\\'s Stenographer,\" as author of a widely translated socialist propaganda pamphlet regarded as a classic of the genre, \"Shop Talks on Economics,\" and as an assistant editor of the \"International Socialist Review\", one of the most influential American socialist magazines of the first two decades of the 20th Century.']], ['Mark Bin Bakar', ['Mark Bin Bakar is an Indigenous Australian musician, comedian and radio announcer, writer, director/producer as well as an indigenous rights campaigner based in Broome, in the Kimberley region of Western Australia.', ' He is best known for his radio and television character, the acid-tongued Mary Geddarrdyu or Mary G, who has gained somewhat of a national cult following and has been described as a Dame Edna Everage in thongs.', ' In character Mary G has hosted a radio program and hosted a variety show broadcast nationally on SBS Television.']], ['Michael and Mary', ['Michael and Mary was a 1931 British drama film directed by Victor Saville and starring Elizabeth Allan, Edna Best, Frank Lawton, and Herbert Marshall.', ' This was the first of the Edna Best and Herbert Marshall co-starring talkies.', ' It was based on a play of the same name by A. A. Milne.']], ['Michael Poole (producer)', ['Michael \"Mike\" Poole was a Canadian film maker and author.', ' He began his career as a copy runner for the \"Vancouver Sun\" before becoming a reporter.', ' He earned a journalism degree in Virginia, USA, started in the film business in the 1960s and went on to be a television producer for the Canadian Broadcasting Corporation for ten years.', ' He then worked as a freelance filmmaker, spending two decades producing documentaries with the well-known Canadian environmentalist, David Suzuki.', ' His books are \"Romancing Mary Jane: A Year in the Life of a Failed Marijuana Grower\", \"Ragged Islands: A Journey by Canoe Through the Inside Passage\" and \"Rain Before Morning\", a novel about Canadian draft dodgers during World War I.', ' In his retirement Poole lived full-time on the Sunshine Coast, British Columbia, Canada with his wife Carole and his two beloved Labradors.', ' He won the Edna Staebler Award, a Canadian literary award for creative nonfiction, in 1999 for \"Romancing Mary Jane: A Year in the Life of a Failed Marijuana Grower\".', ' He died of prostate cancer at the age of 74 in 2010.']], ['Golandsky Institute', ['The Golandsky Institute is a not-for-profit organization dedicated to the Taubman Approach to piano playing.', ' Led by Edna Golandsky, Artistic Director, the Institute holds an annual symposium at Princeton University and hosts workshops and master classes worldwide.', ' The Golandsky Institute was founded in 2003 by Edna Golandsky, John Bloomfield, Robert Durso, and Mary Moran.', ' It now has a teaching roster of fifteen faculty and associate faculty members as well as thirteen certified teachers from around the globe.']], ['The Making of Maddalena', ['The Making of Maddalena is a 1916 American silent drama film directed by Frank Lloyd and written by L.V. Jefferson based upon a play by Samuel Service and Mary Service.', ' The film stars Edna Goodrich, Forrest Stanley, Howard Davies, John Burton, Mary Mersch, and Colin Chase.', ' The film was released on June 8, 1916, by Paramount Pictures.']], ['Pansexuality', ['Pansexuality, or omnisexuality, is the sexual, romantic or emotional attraction towards people regardless of their sex or gender identity.', ' Pansexual people may refer to themselves as gender-blind, asserting that gender and sex are not determining factors in their romantic or sexual attraction to others.']], ['Edna Henry Lee Turpin', ['Edna Henry Lee Turpin (1867–1952) was an American author.', ' She was born on July 26, 1867, at Echo Hill, Mecklenburg County, Virginia.', ' She was the daughter of Edward Henry Turpin and Petronella Lee Turpin, but her father died of tuberculosis four months before she was born.', ' Two siblings, Mary Wilson Turpin and Edward Henry Turpin both died in infancy before Edna was born.', ' She spent her childhood on the family farm with her mother and her older brother, Henderson Lee Turpin (1861–1957).', ' She began writing at an early age and, during her fifteenth year, her first short story was accepted for publication.']], ['Mary Jane Sherfey', ['Mary Jane Sherfey (1918–1983) was an American psychiatrist and writer on female sexuality, she received her medical degree from Indiana University, where she attended lectures on marriage and sexuality given by Alfred Kinsey.', ' Sherfey had a private practice in New York City and was on the staff of the Payne Whitney Clinic of the New York Hospital – Cornell Medical Center.', ' In 1961, Sherfey’s interest in female biology was intensified when she came upon the inductor theory, which demonstrated that the human embryo is female until hormonally “induced” to become male.', ' Determined to popularize a fact that had lain in neglect since its discovery in the 1950s, Sherfey began researching the subject and familiarizing herself with a variety of disciplines, including embryology, anatomy, primatology and anthropology.', ' Many of her findings appear in \"The Nature and Evolution of Female Sexuality\", which initially took form as an article contesting the existence of vaginal orgasm, published in the Journal of the American Psychoanalytic Association in 1966.']], ['Mary González', ['Mary Edna González (born October 30, 1983) is an American politician who serves as State Representative of House District 75 in the Texas House of Representatives.', ' She is a Democrat who was elected in November 2012 to represent an area that includes east El Paso County, parts of the city of El Paso and the towns of Socorro, Clint, Fabens, Horizon City, San Elizario and Tornillo.', ' She is also the first openly pansexual elected official in the United States.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.641\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a79c1185542994bb9457065', 'answer': 'Mark David Hollis', 'question': 'Was Kanako Momota or Mark Hollis born first?', 'supporting_facts': [['Kanako Momota', 0], ['Mark Hollis (musician)', 0]], 'context': [['Laughing Stock (album)', ['Laughing Stock is the fifth and final studio album by British post-rock band Talk Talk.', ' Following on from their previous release \"Spirit of Eden\" (1988), and the departure of bassist Paul Webb, which reduced the band to the duo of Mark Hollis and Lee Harris, Talk Talk acrimoniously left EMI and signed to the jazz-based Verve Records, and recorded \"Laughing Stock\" at Wessex Sound Studios, London, with producer Tim Friese-Greene and engineer Phill Brown from September 1990 to April 1991.']], ['Mark Hollis (album)', ['Mark Hollis is the only solo album by the former Talk Talk frontman Mark Hollis.', ' It was released on Polydor Records on 26 January 1998, then reissued on Pond Life on 13 March 2000.', ' In 2003, the album was released in LP format on Universal Records.', ' Its sound is noted for being extremely sparse and minimal; Allmusic called it \"quite possibly the most quiet and intimate record ever made\".', ' Hollis found inspiration not in the popular music of the day, but rather in 20th-century classical music and jazz from the late fifties and sixties.', ' The album did not mark a return for Hollis to the music industry or live performance: he stated at the time of the album\\'s release that \"There won\\'t be any gig, not even at home in the living room.', ' This material isn\\'t suited to play live.\"']], ['Missing Pieces (Talk Talk album)', ['Missing Pieces is a 2001 compilation album by Talk Talk.', ' The first six tracks are the A- and B-Sides of the three CD singles released in 1991 for their final album \"Laughing Stock\".', ' Four of these are versions of album tracks, with the addition of the otherwise uncollected B-Sides \"Stump\" and \"5:09\".', ' The final track, \"Piano\", was recorded pseudonymously by Mark Hollis (as \"John Cope\", the title of the B-Side of their 1988 single \"I Believe In You\" from the album \"Spirit of Eden\") for the 1998 album \"AV 1\" by Allinson / Brown, which was produced by former Talk Talk producer Phill Brown.', ' According to Hollis, it was designed to cycle indefinitely for a Dave Allinson/Phill Brown art exhibition and is presented twice in a row on the CD.', ' \"Missing Pieces\" was released in 2001 to a generally mixed to positive reception.', ' It is now out of print.']], ['Kanako Nishi', [\"Kanako Nishi (西加南子 , Nishi Kanako , born 13 December 1970, in Kakegawa, Shizuoka, Japan) is a Japanese women's racing cyclist who currently rides for Luminaria.\", \" She won the Japanese National Road Race Championships for women in 2009, the first to win the national title after Miho Oki's eleven straight victories.\"]], ['Mark Hollis (musician)', ['Mark David Hollis (born 4 January 1955) is an English former musician and singer-songwriter.', ' He achieved commercial success in the 1980s as a singer with the synthpop/post-rock band Talk Talk, but retired from the music industry shortly after releasing his 1998 solo debut album.']], ['Mark Hollis (athletic director)', ['Mark Hollis is the athletic director at Michigan State University.', ' He succeeded Ron Mason as athletic director on January 1, 2008.']], ['Such a Shame', ['\"Such a Shame\" is a song written by Mark Hollis for the English band Talk Talk\\'s second album \"It\\'s My Life\" (1984).']], ['Kanako Momota', ['Kanako Momota (百田 夏菜子 , Momota Kanako , born July 12, 1994) is a Japanese idol singer.', ' She is best known as the leader of the female idol group Momoiro Clover Z.', ' Momota was ranked 12th most popular Japanese idol of 2013 by \"Nihon Keizai Shimbun\".', ' She is represented by Stardust Promotion talent agency.']], ['Do Anything You Wanna Do', ['\"Do Anything You Wanna Do\" is a song written by Eddie and the Hot Rods\\' manager Ed Hollis (the brother of Talk Talk\\'s Mark Hollis) and guitarist Graeme Douglas and recorded by the band, although the actual record label credited The Rods as the artist.', ' It reached #9 on the UK Singles Chart in 1977.', ' The song was featured on their 1977 album, \"Life on the Line\".']], [\"It's My Life (Talk Talk song)\", ['\"It\\'s My Life\" is a song by the English new wave band Talk Talk.', \" Written by Mark Hollis and Tim Friese-Greene, it was the title track on the band's second album and released as its first single in January 1984.\", ' It reached #46 in the UK charts, but did better in several other countries, reaching #33 in Germany, #32 in New Zealand, #25 in France and #7 in Italy.', ' It was also a success in North America, entering the Top 40 in both the United States (#31) and Canada (#30).', ' (Notably, it peaked at #1 on the U.S. Hot Dance Club Play chart.)']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.642\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac3a7f75542993915413890', 'answer': 'Orson Welles', 'question': 'The actor who starred in \"The Hearse\" gained worldwide film by starring in whose series of 1940s films?', 'supporting_facts': [['The Hearse', 0], ['Joseph Cotten', 1], ['Joseph Cotten', 2]], 'context': [['Maxim Media Marketing, Inc.', ['Maxim Media Marketing, Inc. is an American-based worldwide film licensing and distribution company founded in 2000 by Darrin Ramage.', ' According to their Company Profile page, their divisions include Brain Damage Films, Midnight Releasing, and Maxim Media International.']], ['Zach Anner', ['Zach Anner (born November 17, 1984 in Buffalo, New York) is an Austin, Texas-based comedian, actor and writer with cerebral palsy who gained worldwide attention with the submission of a video to Oprah Winfrey\\'s \"Search for the Next TV Star\" competition.', ' He won his own TV show on OWN through Oprah\\'s \".\"', ' His wheelchair travel show, \"Rollin\\' With Zach\", premiered December 12, 2011.', ' Recently Zach guest starred on an episode of the TV sitcom \"Speechless\", which is centered around a boy with cerebral palsy.']], ['Julian Shaw', ['Julian Shaw (born 16 December 1985 in Wellington, New Zealand) is an author, filmmaker and actor, best known for directing the 2007 film \"Darling!', ' The Pieter-Dirk Uys Story\", a British Film Institute award-winning documentary about the life of South African political satirist Pieter-Dirk Uys.', ' Shaw is the author of the novel \"Modern Odysseus\", and the creator of several short films including \"Clearing the Air\".', ' He is the director of the 2011 feature documentary \"Cup of Dreams\", about New Zealand’s national Rugby Union team the All Blacks.', \" Shaw also gained worldwide fame in 2011 after starring in an Australian marriage equality advertisement entitled It's Time that has reached millions of people.\"]], ['Joseph Cotten', ['Joseph Cheshire Cotten, Jr. (May 15, 1905 – February 6, 1994) was an American film, stage, radio and television actor.', ' Cotten achieved prominence on Broadway, starring in the original stage productions of \"The Philadelphia Story\" and \"Sabrina Fair\".', ' He first gained worldwide fame in three Orson Welles films: \"Citizen Kane\" (1941), \"The Magnificent Ambersons\" (1942), and \"Journey into Fear\" (1943), for which Cotten was also credited with the screenplay.', ' He went on to become one of the leading Hollywood actors of the 1940s, appearing in films such as \"Shadow of a Doubt\" (1943), \"Love Letters\" (1945), \"Duel in the Sun\" (1946), \"Portrait of Jennie\" (1948), \"The Third Man\" (1949) and \"Niagara\" (1953).', ' One of his final films was Michael Cimino\\'s \"Heaven\\'s Gate\" (1980).']], ['David Devine (director)', ['David Devine is a film director and producer.', ' He is a creator of original content for film, television and digital media.', ' He has been the CEO of both public and private companies.', ' His 19 HBO Original films have received 12 EMMY Award nominations (winning 5) and 22 Canadian Screen Award nominations (winning 6) amongst a total of 80 worldwide film awards.', ' Highlights of his directing career include \"Beethoven Lives Upstairs\", \"Einstein: Light to the Power of Two\", \"Galileo: On the Shoulders of Giants\", \"Degas and the Dancer\", \"Edison: The Wizard of Light\" and \"Bailey\\'s Billion$\".', ' David has also been a hands-on creator and producer of 16 symphonic soundtracks for his films and 6 CDs distributed on the SONY Classical label.', ' David has been focused on addressing important creative, social and cultural issues in his films and digital media throughout his directing and producing career.', ' By January, 2017 his films had been broadcast in over 110 countries and 3.6M DVDs had been sold.', ' 600,000 of these DVDs have been used by teachers in elementary and middle school classrooms from 2004 to 2017 in the U.S. and Canada and over 800,000 DVDs have been sold to U.S. parents of elementary and middle school aged home schoolers.']], ['The Hearse', ['The Hearse is a 1980 American horror film starring Trish Van Devere and Joseph Cotten.']], ['Mark Billingham', ['Mark Philip David Billingham (born 2 July 1961) is an English novelist, actor, television screenwriter and comedian whose series of \"Tom Thorne\" crime novels are best-sellers in that particular genre.']], ['Nick Raider', ['Nick Raider is a fictional comic detective whose series was published from 1988 to 2005 by Sergio Bonelli Editore in Italy.', ' Graphically inspired by the American actor Robert Mitchum, it was created by writer Claudio Nizzi and artist Giampiero Casertano.', ' His partner Marvin Brown was inspired by Eddie Murphy.', \" Nick Raider's stories are set in New York City.\"]], ['Ringo Starr', ['Richard Starkey, {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} (born 7 July 1940), known professionally as Ringo Starr, is an English drummer, singer, songwriter and actor who gained worldwide fame as the drummer for the Beatles.', ' He occasionally sang lead vocals, usually for one song on an album, including \"With a Little Help from My Friends\", \"Yellow Submarine\", \"Good Night\", and their cover of \"Act Naturally\".', ' He also wrote the Beatles\\' songs \"Don\\'t Pass Me By\" and \"Octopus\\'s Garden\", and is credited as a co-writer of others, including \"What Goes On\" and \"Flying\".']], ['Harrison Ford', ['Harrison Ford (born July 13, 1942) is an American actor and film producer.', ' He gained worldwide fame for his starring roles as Han Solo in the \"Star Wars\" film series and as the title character of the \"Indiana Jones\" film series.', ' Ford is also known for his roles as Rick Deckard in the neo-noir dystopian science fiction film \"Blade Runner\" (1982); John Book in the thriller \"Witness\" (1985), for which he was nominated for the Academy Award for Best Actor; and Jack Ryan in the action films \"Patriot Games\" (1992) and \"Clear and Present Danger\" (1994).']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.643\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ade17d255429939a52fe80e', 'answer': 'yes', 'question': 'Are both the Wire Fox Terrier and West Siberian Laika dog breeds?', 'supporting_facts': [['Wire Fox Terrier', 0], ['West Siberian Laika', 0]], 'context': [['Kathleen Pelham-Clinton, Duchess of Newcastle', ['Kathleen Florence May Pelham-Clinton, Duchess of Newcastle OBE (1872 – 1 June 1955), was a well-known conformation show judge and dog breeder who influenced the Borzoi and Wire Fox Terrier breeds.']], ['Signal Circuit of Halleston', ['Signal Circuit of Halleston was a Wire Fox Terrier and winner of the 1926 Westminster Kennel Club Dog Show.', ' This was the 50th Westminster show, and the fourth Fox Terrier to win best in show.']], ['Canine degenerative myelopathy', ['Canine degenerative myelopathy, also known as chronic degenerative radiculomyelopathy, is an incurable, progressive disease of the canine spinal cord that is similar in many ways to amyotrophic lateral sclerosis (ALS).', ' Onset is typically after the age of 7 years and it is seen most frequently in the German shepherd dog, Pembroke Welsh corgi, and boxer dog, though the disorder is strongly associated with a gene mutation in SOD1 that has been found in 43 breeds as of 2008, including the wire fox terrier, Chesapeake Bay retriever, Rhodesian ridgeback, and Cardigan Welsh corgi.', ' Progressive weakness and incoordination of the rear limbs are often the first signs seen in affected dogs, with progression over time to complete paralysis.', ' Myelin is an insulating sheath around neurons in the spinal cord.', ' One proposed cause of degenerative myelopathy is that the immune system attacks this sheath, breaking it down.', ' This results in a loss of communication between nerves in lower body of the animal and the brain.']], ['West Siberian Laika', ['The West Siberian Laika or \"WSL\", is a breed of hunting dog and a breed of spitz type.', ' Russian publications indicate that the term West Siberian Laika loosely applied to hunting dogs originating with the Mansi and Khanty people in Ural and West Siberia, but there were no standards or registrations of WSL as such until 1930.', ' Then WWll disrupted it for a while, but \"systematic breeding with registrations\" resumed after the war ended, in 1946.', ' This was the time the breed began taking modern shape.', ' Before that hunters only knew of Mansi Laika and Khanty Laika.', ' In early 1960 many hunters in Ural still preferred the term Mansi Laika, when speaking of West Siberian Laika.', ' In Russian language, the term Laika originated from the word \"layat\" that means to bark.', ' The word Laika simply means \"barker\".', ' Any hunting Laika is a bark pointer (pointing at animal of interest by barking and staying with the animal ).', ' It is a versatile dog depending on use and environment, but in certain parts of the country they have become more specialized.']], ['Teddy Roosevelt Terrier', ['The Teddy Roosevelt Terrier is a small to medium-sized American hunting terrier.', ' Lower-set with shorter legs, more muscular, and heavier bone density than its cousin the American Rat Terrier.', ' There is much diversity in the history of the Teddy Roosevelt Terrier breed and it shares a common early history with the American Rat Terrier, Fox Paulistinha and Tenterfield Terrier.', ' It is said the Rat Terrier background stems from the terriers or other dogs that were brought over by early English and other working class immigrants.', ' Since the breed was a farm, hunting and utility dog there was little to no planned breeding other than breeding dogs with agreeable traits to each other in order to produce the desired work ethic in the dog.', \" It is assumed that the Feist (dog), Bull Terrier, Smooth Fox Terrier, Manchester Terrier, Whippet, Italian Greyhound, the now extinct English White Terrier, Turnspit dog and or Wry Legged Terrier all share in the Teddy Roosevelt Terrier's ancestry.\", ' These early Ratting Terriers were then most likely bred to the Beagle or Beagle cross bred dogs (for increased scenting ability) and other dogs.', ' Maximizing the influences from these various breeds provides the modern Teddy Roosevelt Terrier with a keen sense of awareness and prey drive, an acute sense of smell and a very high intellect.', ' Although they tend to be aloof with strangers they are devoted companion dogs with a strong desire to please and be near their owners side at all times.']], ['Caesar (dog)', ['Caesar (1898–1914) was a Wire Fox Terrier owned by King Edward VII.', ' He was bred in the kennels of Kathleen, Duchess of Newcastle, and became the constant companion of the King.', \" After the King's death in 1910, the dog attended the funeral and walked in the procession in prominence ahead of nine kings and other heads of state.\", ' Caesar has been the subject of paintings, and a hand crafted hardstone model created by the House of Fabergé.']], ['Fox Terrier', ['Fox Terriers are two different breeds of the terrier dog type: the Smooth Fox Terrier and the Wire Fox Terrier.', ' Both of these breeds originated in the 19th century from a handful of dogs who are descended from earlier varieties of British terriers, and are related to other modern white terrier breeds.', ' In addition, a number of breeds have diverged from these two main types of fox terrier and have been recognised separately, including the Jack Russell Terrier, Miniature Fox Terrier and Rat Terrier.', ' The Wire and Smooth Fox Terriers share similar characteristics, the main differences being in the coat and markings.', ' They have been successful in conformation shows, more prominently in America than their homeland.']], ['Skippy (dog)', ['Skippy (also known as Asta, born 1931 or 1932; retired 1941) was a Wire Fox Terrier dog actor who appeared in dozens of movies during the 1930s.', ' Skippy is best known for the role of the pet dog \"Asta\" in the 1934 detective comedy \"The Thin Man\", starring William Powell and Myrna Loy.', ' Due to the popularity of the role, Skippy is sometimes credited as Asta in public and in other films.']], ['Wire Fox Terrier', ['The Wire Fox Terrier is a breed of dog, one of many terrier breeds.', ' It is a fox terrier, and although it bears a resemblance to the smooth fox terrier, they are believed to have been developed separately.']], ['Old Jock', ['Old Jock (1859–1871), was a Fox Terrier famous during the late 19th and early 20th centuries.', ' A mostly white dog, he ran briefly with a hunting kennel before becoming a show dog, most notably with a victory at the show which popularised the Fox Terrier.', ' His main show rivalry was with a dog named Tartar, and along with a dog named Trap, the three were popular sires of the Fox Terrier breed.', ' He was also involved in the early formation of the Jack Russell Terrier and the Dandie Dinmont Terrier breeds.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.643\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac249f55542992f1f2b3831', 'answer': 'Australian', 'question': 'What nationality was the inventor of the Alexander Technique for retraining the mind and body to avoid muscular tension?', 'supporting_facts': [['F. Matthias Alexander', 0], ['Alexander Technique', 0]], 'context': [['Z-plasty', ['Z-plasty is a versatile plastic surgery technique that is used to improve the functional and cosmetic appearance of scars.', ' It can elongate a contracted scar or rotate the scar tension line.', ' The middle line of the Z-shaped incision (the central element) is made along the line of greatest tension or contraction, and triangular flaps are raised on opposite sides of the two ends and then transposed.', ' The length and angle of each flap are usually the same to avoid mismatched flaps that may be difficult to close.', ' Some possible complications of Z-plasty include flap necrosis, haematoma (blood clot) formation under the flaps, wound infection, trapdoor effect and sloughing (necrosis) of the flap caused by wound tension and inadequate blood supply.']], ['Progressive muscle relaxation', ['Progressive muscle relaxation (PMR) is a non-pharmacological method of deep muscle relaxation, based on the premise that muscle tension is the body’s psychological response to anxiety-provoking thoughts and that muscle relaxation blocks anxiety.', ' The technique involves learning to monitor the tension in specific muscle groups by first tensing each muscle group.', ' This tension is then released, as attention is directed towards the differences felt during tension and relaxation.']], ['Mitzvah Technique', ['The Mitzvah Technique is focused on dealing with body mechanics in a state of motion.', ' It is a development of the Alexander Technique, the Feldenkrais Method and health-oriented work on musculoskeletal problems and stress diseases.', ' Each of these techniques is based on correcting common postural faults by addressing the neuromuscular system through postural re-education.', ' Yet, the Mitzvah Technique encompasses both a unique philosophy and a set of procedures.', ' This includes the discipline, exercises, and the work that Mitzvah Technique practitioners do with their hands.']], ['F. Matthias Alexander', ['Frederick Matthias Alexander (20 January 1869 – 10 October 1955) was an Australian actor who developed the Alexander Technique, an educational process applied to recognize and overcome reactive, habitual limitations in movement and thinking.']], ['Alexander Technique', ['The Alexander Technique (A.T.), named after Frederick Matthias Alexander, is an educational process that attempts to develop the ability to avoid unnecessary muscular tension by retraining physical movement reactions.', \" Alexander believed the individual's spatial self-awareness was related to psychological conditioning; questionably trained foundation habits of posture can be unsuitably added into procedural skills.\", ' Alexander observed that those whose goals justified it necessary to have trained themselves to overcompensate could not trust their judgment of physical orientation and required effort, (their \"sensory appreciation.\")', '.']], ['Rosen Method Bodywork', ['Rosen Method Bodywork (or Rosen Method) is a type of Complementary and alternative medicine.', ' This bodywork, described as \"psycho-somatic\", claims to help integrate one\\'s bodily and emotional/mental experience while identifying unconscious patterns of muscular holding, feeling, and behavior.', \" The main theory underpinning this method is that a person protects themselves from past painful experiences through the body, separating one from one's true self.\", ' This alleged protection is said to be experienced most frequently as chronic musculoskeletal pain and tension, and purportedly can be observed by the bodywork practitioners as restricted patterns of movement and posture, muscular tension, or shortness of breath.', ' Rosen Method Bodywork purports to integrate the body, mind, emotions and spirit; and unlock the unconscious.']], ['Bioenergetic analysis', ['Bioenergetic analysis is a form of body psychotherapy (body-oriented Reichian psychotherapy), based upon the work of Wilhelm Reich.', ' It can also be termed as a very specific kind of body psychotherapy which is based upon the continuity between body and mind.', ' This form of body psychotherapy adds a number of innovations to the classic methods, these innovations include emphasis on the importance of grounding (i.e. being in strong contact with the ground through feet and legs) and on psychoanalytic theories such as transference, countertransference, dreams, slips of the tongue and Oedipal issues.', ' It also places even greater emphasis on sexual fulfilment than Reichian psychotherapy.', ' It was developed by Alexander Lowen and John Pierrakos, both patients and students of Reich.', ' The idea behind current bioenergetic practice is that blocks to emotional expression and wellness are revealed and expressed in the body as chronic muscle tensions which are often subconscious.', ' The blocks are treated by combining bioenergetically designed physical exercises, affective expressions and palpation of the muscular tensions.']], ['Anxiety', ['Anxiety is an emotion characterized by an unpleasant state of inner turmoil, often accompanied by nervous behavior, such as pacing back and forth, somatic complaints, and rumination.', ' It is the subjectively unpleasant feelings of dread over anticipated events, such as the feeling of imminent death.', ' Anxiety is not the same as fear, which is a response to a real or perceived immediate threat, whereas anxiety is the expectation of future threat.', ' Anxiety is a feeling of uneasiness and worry, usually generalized and unfocused as an overreaction to a situation that is only subjectively seen as menacing.', ' It is often accompanied by muscular tension, restlessness, fatigue and problems in concentration.', ' Anxiety can be appropriate, but when experienced regularly the individual may suffer from an anxiety disorder.']], ['Neil Greenberg (choreographer)', ['Greenberg\\'s work is characterized by a \"choreographic lexicon that integrates kinesthetic, emotional, and cognitive ways of knowing and representing the world and the self\".)', ' Within this framework, Greenberg\\'s work deals with the queer male body dancing, a theme that has been implicit throughout his dance making and began to become explicitly identified starting with \" Quartet for Three Gay Men \" (2006) and extending into his subsequent dances.', ' Much of the movement in his choreography is based on improvisation and is reflective of his in depth study of somatic techniques, such as Body/Mind Centering, Klein Technique, and Alexander Technique.', \" However, Cunningham's influence can be seen in Greenberg's practice of working with the non-fiction of the body on stage and combining different elements, such as movement, projection, and sound, that leave the responsibility of meaning-making up to the audience.\", ' Greenberg has created over 20 works for Dance by Neil Greenberg, as well as additional commissions for Mikhail Baryshnikov\\'s White Oak Dance Project, Ricochet Dance Company, John Jesurun\\'s \"Chang in a Void Moon\", and various colleges across the country.']], ['Aiki (martial arts principle)', [\"Aiki from a Japanese budo term, at its most basic is a principle that allows a conditioned practitioner to negate or redirect an opponent's power on contact.\", ' When applied, the Aiki practitioner controls the actions of the attacker with minimal effort and with a distinct absence of muscular tension usually associated with physical effort.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.645\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a80e5f5554299260e20a1ae', 'answer': '5.3 million', 'question': 'How many of the 12 million immigrants to the United States through Ellis Island were Italian immigrants?', 'supporting_facts': [['History of Italian Americans in Boston', 0], ['Ellis Island', 0]], 'context': [['History of Italian Americans in Boston', ['Not all of the 5.3 million Italians who immigrated to the United States between 1820 and 1978 came through Ellis Island.', ' Many came through other ports, including the Port of Boston.', ' Exactly how many stayed in Boston is not known, but it was enough to make Italians the second largest ancestry group in Boston, after the Irish.', ' Most settled initially in the North End; others settled in East Boston, the West End, Roxbury, and other neighborhoods.']], ['Ellis Island Immigrant Hospital', ['The Ellis Island Immigrant Hospital, also known as USPHS Hospital #43, was the United States’ first public health hospital, opened in 1902 and operating as a hospital until 1930.', ' Constructed in phases, the facility encompassed both a general hospital and a separate pavilion style contagious disease hospital.', ' The hospital served as a detention facility for new immigrants who were deemed unfit to enter the United States after their arrival; immigrants would either be released from the hospital to go on to a new life in America or sent back to their home countries.', ' The hospital was one of the largest public health hospitals in United States history and is still viewed today as an extraordinary endeavor in the public health field.', ' While the monument is managed by the National Park Service as part of the National Parks of New York Harbor office, the south side of Ellis Island has been off-limits to the general public since its closing.', ' Efforts to restore the hospital buildings and others on the island are being made by government partner Save Ellis Island.', ' In October 2014, the hospital opened to the public for small group hard hat tours.']], ['History of soccer in the United States', ['The history of soccer in the United States has numerous different roots.', ' The modern-day game, is often considered to have been brought to the United States through Ellis Island during the 1870s.', ' However, recent research has shown that the modern game entered America in the 1850s through New Orleans when Scottish, Irish, German and Italian immigrants brought the game with them.', ' It was in New Orleans that some of the first organized games that used modern English rules were held.']], ['Community organizing in immigrant communities', ['Many immigrant communities in the United States are engaged in community organizing activities.', ' Of over 50 million immigrants living in the United States many may experience exploitation in the workforce and different forms of discrimination and challenges in their lives.', ' Many voluntary associations that seek to meet the needs of immigrants utilize community organizing methods aiming to mobilize and empower them and advocate for them.']], ['Pier 21', ['Pier 21 was an ocean liner terminal and immigration shed from 1928 to 1971 in Halifax, Nova Scotia, Canada.', ' Over one million immigrants came to Canada through Pier 21 and it is the last surviving seaport immigration facility in Canada.', ' The facility is often compared to the landmark American immigration gateway Ellis Island.', ' The former immigration facility is now occupied by the Canadian Museum of Immigration, the Nova Scotia College of Art and Design as well as various retail and studio tenants.']], ['Ellis Island', [\"Ellis Island, in Upper New York Bay, was the gateway for over 12 million immigrants to the United States as the nation's busiest immigrant inspection station for over sixty years from 1892 until 1954.\", ' The island was greatly expanded with land reclamation between 1892 and 1934.', ' Before that, the much smaller original island was the site of Fort Gibson and later a naval magazine.', ' The island was made part of the Statue of Liberty National Monument in 1965, and has hosted a museum of immigration since 1990.']], ['Ellis Island (miniseries)', ['Ellis Island is a television miniseries, filmed in the United Kingdom, broadcast in three parts in 1984 on the CBS television network.', ' The screenplay was co-written by Fred Mustard Stewart, adapted from his 1983 novel of the same title.', ' The series tells the story of several immigrants from the late 19th century until the early 1910s, trying to achieve the American Dream and arriving on Ellis Island, hoping for a better life.', ' \"Ellis Island\" highlighted numerous important events which occurred up to and during World War I, and many of the characters are based on real persons, such as Irving Berlin.', ' Ellis Island marks the final appearance of Richard Burton and the series was dedicated to his memory.']], ['Ellis Island Special', ['An Ellis Island Special is a family name that is perceived or labeled, incorrectly, as having been altered by immigration officials at the Ellis Island Immigration Station, when a family reached the United States, typically from Europe in the 19th and early 20th centuries.', ' In popular thought, some family lore, and literary fiction, some family names were seen as having been shortened by immigration officials for ease of pronunciation or record-keeping, or lack of understanding of the true name—even though name changes were made by the immigrants themselves at other times.', ' Among the family names that are perceived as being Ellis Island Specials are some that were supposedly more identifiably Jewish, resulting in last names that were not identifiably so.']], ['Ellis Island: The Dream of America', ['Ellis Island: The Dream of America is a work for actors and orchestra with projected images by American composer Peter Boyer, composed in 2001-02, commissioned by the Bushnell Center for the Performing Arts in Hartford, Connecticut.', ' The work combines first-person narrations of seven immigrants who entered the United States through Ellis Island between 1910 and 1940, selected by Boyer from the Ellis Island Oral History Project, with Boyer’s original orchestral music.', ' The work has received over 170 performances by more than 75 orchestras.', ' A recording of the work released on the Naxos record label was nominated for a Grammy Award for Best Classical Contemporary Composition in the 48th annual Grammy Awards.', ' In April 2017, performances by Pacific Symphony of \"Ellis Island: The Dream of America\" were filmed for broadcast on PBS’ \"Great Performances\" series in the 2017-18 broadcast season.']], ['Stephen Wilkes', ['Stephen Wilkes is an American photographer known foremost for his series of abandoned structures such as at Ellis Island and the former Bethlehem Steel factory both which he has captured as a lost world caught in a sort of visual amber.', ' The Wilkes photographic essay on Ellis Island \"Ellis Island Ghosts\" helped to raise six million dollars from the United States Congress for the preservation of the structures on the south side of the island, including the former hospital for infectious diseases.', ' His fine art and photo-journalism have been featured in such publications as Vanity Fair, Sports Illustrated, and The New York Times Magazine.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.645\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8331f15542993344745fe8', 'answer': 'film lyricist', 'question': 'What other jobs did the man that co-produced Gyara Hazar Ladkian have?', 'supporting_facts': [['Gyara Hazar Ladkian', 1], ['Ali Sardar Jafri', 1]], 'context': [['Steve Jobs: The Man in the Machine', ['Steve Jobs: The Man in the Machine is a documentary film about Steve Jobs directed and produced by Alex Gibney.', ' After a few festival showings, the film was released to the Internet on September 4, 2015 and in limited release to theaters on September 19, 2015.']], ['Give a Man a Job', ['Give a Man a Job is a short film produced in 1933 in conjunction with the National Recovery Administration in which audience members were encouraged to offer jobs to the unemployed in the midst of the Great Depression.', ' The film featured Jimmy Durante explaining to an audience through a comic song how they could generate employment.', ' Upon learning that a banker drives his own car, Durante suggests that he \"hire a chauffeur / And keep a good man from becoming a loafer.\"', ' He also has an exchange with Moe Howard playing an exterminator, suggesting that NRA stands for \"No Rats Allowed.\"', ' The film closes with an image of President Franklin Roosevelt and the words \"If the old name of Roosevelt / Makes your old heart throb / Then take this message, straight from the President / And give a man a job!\"']], ['The Lobster', ['The Lobster is a 2015 absurdist dystopian black comedy film directed, co-written, and co-produced by Yorgos Lanthimos, co-produced by Ceci Dempsy, Ed Guiney, and Lee Magiday, and co-written by Efthimis Filippou.', \" In the film's setting, single people are given 45 days to find a romantic partner or otherwise be turned into animals.\", ' It stars Colin Farrell as a newly-single man trying to find someone so he can remain human, and Rachel Weisz as a woman with whom he attempts to form a relationship.', ' The film is co-produced by Ireland, United Kingdom, Greece, France and the Netherlands.']], ['Gyara Hazar Ladkian', ['Gyara Hazar Ladkian (Eleven Thousand Girls ) is a 1962 Hindi romantic social drama film directed by K. A. Abbas.', ' The film was co-produced by the poet Ali Sardar Jafri, who also helped co-write the story with Abbas.']], ['Jeremy Theobald', ['Jeremy Theobald is a British actor best known for his portrayal of \"The Young Man\", the main character in Christopher Nolan\\'s 1998 major picture debut \"Following\", and for which Theobald was also a producer, Filming was scheduled around their day jobs.', ' Jonathan Romney, writing in the \"New Statesman\", noted that \"Nolan and his cast are terrific finds: I wouldn\\'t normally say this to struggling artists, but they might want to give up their day jobs.\"']], ['This Is 40', ['This Is 40 is a 2012 American comedy film written, co-produced and directed by Judd Apatow, and starring Paul Rudd and Leslie Mann.', ' It is the spin-off sequel of \"Knocked Up\", which starred Seth Rogen and Katherine Heigl.', ' Filming was conducted in mid-2011, and the film was released in North America on December 21, 2012.', ' The film follows the lives of middle-aged married couple Pete and Debbie as they each turn 40, with their jobs and daughters adding stress to their relationship.']], ['Mary Agnes Donoghue', ['Mary Agnes Donoghue (born 1942/1943) is an American screenwriter and director.', ' Following early jobs as a secretary and short story writer, Donoghue\\'s first writing credit was the 1984 film \"The Buddy System\".', ' She went on to pen the screenplays for \"Beaches\" (1988) and \"Paradise\" (1991), which was also her directorial debut.', ' Donoghue co-wrote and co-produced \"Deceived\" (1991) and two year later, her first play, \"Me and Mamie O\\'Rourke\", made its debut at the Strand Theatre in London.', ' In the 2000s, Donoghue wrote the screenplay for \"White Oleander\" (2002) and co-wrote \"Veronica Guerin\" (2003) with Carol Doyle.', ' In 2013, Donoghue wrote and directed \"Jenny\\'s Wedding\".']], ['Ali Sardar Jafri', ['Ali Sardar Jafri (29 November 1913 – 1 August 2000) was a prolific and versatile Urdu writer from India.', ' He was also a poet, critic and film lyricist.']], ['Salaryman', ['Salaryman (サラリーマン, , \"Sararīman\" , salaried man) refers to a man whose income is salary based, particularly those working for corporations.', ' It has gradually become accepted in Anglophone countries as a noun for a Japanese white-collar worker or businessman.', ' The term salaryman refers exclusively to men; for women the term career woman or, for lower prestige jobs, office lady is used.']], ['Women in Belgium', ['Women in Belgium are European women who live in or are from Belgium.', ' Generation after generation, Belgian women are able to close the \"occupational gender gap\".', ' In younger generations, this is due to the increasing availability of \"part-time jobs in services\" for women.', ' In 1999, the average earnings of a Belgian woman was 91 percent of the salary of a Belgian man.', ' When not doing part-time jobs, Belgian women still \"do more of the domestic work\", depending on the agreement between female and male partners.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.645\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a728f505542992359bc30f5', 'answer': 'Dumb and Dumber', 'question': 'Gail Matthius co-anchored the Weekend Update segment of \"Saturday Night Live\" with the actor who played the villain Nicholas Andre in what movie?', 'supporting_facts': [['Gail Matthius', 1], ['Charles Rocket', 1]], 'context': [['A. Whitney Brown', ['Alan Whitney Brown (born July 8, 1952) is an American writer and comedian best known for work on \"Saturday Night Live\" in the 1980s.', ' In addition to writing for the program, he appeared opposite Dennis Miller in a biting satirical Weekend Update commentary segment called \"The Big Picture.\"', \" He won a 1988 Emmy Award for Outstanding Writing in a Variety or Music Program, along with Al Franken, Tom Davis, Phil Hartman, Mike Myers, Lorne Michaels and Conan O'Brien.\", ' He was also one of the original correspondents on Comedy Central\\'s \"The Daily Show\" from 1996 to 1998.']], ['Charles Rocket', ['Charles Adams Claverie (August 28, 1949  – October 7, 2005), known by such stage names as Charlie Hamburger, Charlie Kennedy, and, most famously, Charles Rocket, was an American actor.', ' He was best known for his tenure as a cast member on \"Saturday Night Live\", for his role as the villain Nicholas Andre in the film \"Dumb and Dumber\", and for his appearance as Dave Dennison, Max and Dani Dennison\\'s father in Disney\\'s \"Hocus Pocus\".']], ['Saturday Night Live Weekend Update Thursday', ['Saturday Night Live Weekend Update Thursday is an American limited-run series broadcast on NBC.', ' It is a political satire news show spin-off from \"Saturday Night Live\", featuring that show\\'s \"Weekend Update\" segment.', ' It initially ran for three 30-minute episodes in October 2008, during the lead-up to the 2008 United States presidential election.']], ['Gail Matthius', ['Gail Matthius (born December 14, 1953) is an American actress, voice actress and comedian.', ' She was a cast member of NBC\\'s \"Saturday Night Live\" during its critical and ratings low point at the time (the 1980–1981 season headed by Jean Doumanian), and co-anchored the Weekend Update segment with Charles Rocket in 1981.']], ['Roseanne Roseannadanna', ['Roseanne Roseannadanna is one of several recurring characters created by Gilda Radner, who appeared on \"Weekend Update\" in the early seasons of \"Saturday Night Live\" (\"SNL\"), which aired on the NBC network.', \" She was the segment's consumer affairs reporter who, like an earlier Radner character Emily Litella, editorialized on current issues, only to go off-topic before interrupted by the anchor.\", \" Unlike Litella's meek and apologetic character, Roseannadanna was brash and tactless.\", ' The character was based on Rose Ann Scamardella, a former anchorwoman on WABC-TV\\'s \"Eyewitness News\" in New York City.', \" The character also appeared later in Radner's live one-woman shows.\"]], ['Herb Sargent', ['Herbert Sargent (July 15, 1923 – May 6, 2005) was an American television writer, a producer for such comedy shows as \"The Tonight Show\" and \"Saturday Night Live\", and a screenwriter (\"Bye Bye Braverman\").', ' During his tenure at \"Saturday Night Live\", he and Chevy Chase created Weekend Update, the longest-running sketch in the show\\'s history, and one of the longest running sketches on television.']], ['Weekend Update', ['Weekend Update is a \"Saturday Night Live\" sketch that comments on and parodies .', \" It is the show's longest-running recurring sketch, having been on since the show's first broadcast, and is typically presented in the middle of the show immediately after the first musical performance.\", ' One or two of the players are cast in the role of news anchor, presenting gag news items based on current events and acting as hosts for occasional editorials, commentaries, or other performances by other cast members or guests.', ' Chevy Chase has claimed that \"Weekend Update\" which he started as anchor in 1975 paved the way for comedic news shows like \"The Daily Show\" and \"The Colbert Report\".']], ['Seth Meyers', ['Seth Adam Meyers (born December 28, 1973) is an American comedian, writer, political commentator, actor, and television host.', ' He hosts \"Late Night with Seth Meyers\", a late-night talk show that airs on NBC.', ' Prior to that, he was a head writer for NBC\\'s \"Saturday Night Live\" (2001–2014) and hosted the show\\'s news parody segment, \"Weekend Update\".']], ['Chevy Chase', ['Cornelius Crane \"Chevy\" Chase ( ; born October 8, 1943) is an American actor and comedian.', ' Born into a prominent New York family, he worked a variety of jobs before moving into comedy and began acting with \"National Lampoon\".', ' He became a key cast member in the debut season of \"Saturday Night Live\", where his recurring \"Weekend Update\" segment soon became a staple of the show.', ' As both a performer and writer, he earned three Primetime Emmy Awards out of five nominations.']], ['The Chanukah Song', ['\"The Chanukah Song\" is a novelty song written by comedian Adam Sandler with \"Saturday Night Live\" writers Lewis Morton and Ian Maxtone-Graham and originally performed by Sandler on \"Saturday Night Live\"\\' s Weekend Update on December 3, 1994.', ' Sandler subsequently performed the song as part of his stand-up act, later updating it with new lyrics.', \" All variations center on the theme of Jewish children feeling alienated during the Christmas season, and Sandler's listing of Jewish celebrities (both real and fictional) as a way of sympathizing with their situation.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.646\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae3281a5542991a06ce9939', 'answer': 'no', 'question': 'Are both Frank Lloyd and Olivier Assayas American film directors?', 'supporting_facts': [['Frank Lloyd', 0], ['Olivier Assayas', 0]], 'context': [['Irma Vep', ['Irma Vep is a 1996 film directed by the French director Olivier Assayas, starring Hong Kong actress Maggie Cheung (playing herself) in a story about the disasters that result as a middle-aged French film director (played by Jean-Pierre Léaud) attempts to remake Louis Feuillade\\'s classic silent film serial \"Les vampires\".', ' Taking place as it does largely through the eyes of a foreigner (Cheung), it is also a meditation on the state of the French film industry at that time.']], ['Something in the Air (2012 film)', ['Something in the Air (French: Après mai ) is a 2012 French drama film written and directed by Olivier Assayas.', ' The film was selected to compete for the Golden Lion at the 69th Venice International Film Festival.', ' Assayas won the Osella for Best Screenplay.']], ['Cold Water (film)', ['Cold Water (French: \"L\\'eau froide\" ) is a 1994 French film written and directed by Olivier Assayas.', ' About two troubled teenagers in France during the early 1970s, the film was screened in the Un Certain Regard section at the 1994 Cannes Film Festival.']], ['Frank Lloyd', ['Frank William George Lloyd (2 February 1886 – 10 August 1960) was an American film director, scriptwriter and producer.', ' He was among the founders of the Academy of Motion Picture Arts and Sciences, and was its president from 1934-35.']], ['Graduation (2016 film)', ['Graduation (Romanian: Bacalaureat ; working title: \"Family Photos\") is a 2016 Romanian-language drama film produced, written, and directed by Cristian Mungiu and starring Adrian Titieni and Maria-Victoria Dragus.', ' Set in a small Romanian town, the film focuses on a doctor.', \" It was selected to compete for the Palme d'Or at the 2016 Cannes Film Festival.\", ' At Cannes, Mungiu shared the Best Director Award with Olivier Assayas for his film \"Personal Shopper\".']], ['Jorge Arriagada', ['Jorge Arriagada (born 1943) is a Chilean film composer.', ' He is perhaps best known for his long-term collaboration with director Raúl Ruiz.', ' He has also worked with directors Patricio Guzman, Barbet Schroeder and Olivier Assayas.']], ['Zeitgeist Films', ['Zeitgeist Films is an American independent film distributor based in New York City founded in 1988 by co-Presidents Nancy Gerstman and Emily Russo.', ' Films distributed by Zeitgeist are strongly auteur-driven by directors such as Christopher Nolan, Guy Maddin, Atom Egoyan, Todd Haynes, Nuri Bilge Ceylan, Olivier Assayas, Abbas Kiarostami, Deepa Mehta, Jan Švankmajer and the Brothers Quay.', ' The expansive Zeitgeist film library includes \"Trouble the Water\", \"The Corporation\", \"Jellyfish\", \"Examined Life\", \"Into Great Silence\", Ten and Irma Vep.', ' In June 2008, the MoMA honored two decades of Zeitgeist successes with a month-long, twenty film retrospective entitled \"Zeitgeist: The Films of Our Time\", exhibiting the distributor\\'s twenty most critically acclaimed, intellectually stimulating titles.']], [\"Eddie's House\", [\"Eddie's House was a doghouse designed by Frank Lloyd Wright for the Berger family of San Anselmo, California, to be used by their dog Eddie.\", \" Wright designed Eddie's House to be in keeping with the family's home, known as the Robert Berger House, which he had previously designed.\", ' The plans for the doghouse were completed by Wright in 1957, and the four square foot triangular house was built in 1963.', ' In 1973 Eddie\\'s House was removed and thrown away, but in 2010 Jim and Eric Berger, sons of Robert Berger, rebuilt Eddie\\'s House from the original plans for a segment in \"Romanza\", a documentary film by Michael Miner about Frank Lloyd Wright\\'s architectural works in California.', ' The doghouse remains the smallest structure Frank Lloyd Wright ever designed.']], ['Clouds of Sils Maria', ['Clouds of Sils Maria (known simply as Sils Maria in some territories) is a 2014 drama film written and directed by Olivier Assayas, and starring Juliette Binoche, Kristen Stewart, and Chloë Grace Moretz.', ' The film is a French-German-Swiss co-production.', ' Principal photography took place from August to October 2013, with most of the filming taking place in Sils Maria, Switzerland.', ' The film follows an established middle-aged actress (Binoche) who is cast as the older lover in a romantic lesbian drama opposite an upstart young starlet (Moretz).', ' She is overcome with personal insecurities and professional jealousies—all while sexual tension simmers between her and her personal assistant (Stewart).', ' The screenplay was written with Binoche in mind and incorporates elements from her life into the plot.']], ['Olivier Assayas', ['Olivier Assayas (born 25 January 1955) is a French film director, screenwriter and film critic.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.647\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5aba9d7955429901930fa85f', 'answer': 'Reese Witherspoon', 'question': 'Which actor/actress from Overnight Delivery was born in New Orleans?', 'supporting_facts': [['Overnight Delivery', 2], ['Reese Witherspoon', 1]], 'context': [['LSO (company)', ['LSO, formerly Lone Star Overnight, is a regional shipping carrier that focuses on overnight delivery, utilizing both air and ground transportation, to every address in Texas, southeastern New Mexico and all major metro markets in Oklahoma, Louisiana, Arkansas, Alabama & Tennessee, plus the country of Mexico.', ' Headquartered in Austin, Texas, LSO offers hundreds of drop box locations throughout its service area.', ' Like its competitors, LSO’s brand distinguishes itself with a signature shade of blue.']], ['Ron Bechet', ['Ron Bechet is a visual artist who works in the traditional mediums of drawing and painting.', ' Bechet was born in New Orleans, LA and currently lives and works in New Orleans, as well.', ' He completed his BFA at University of New Orleans in Louisiana , and received an MFA from Yale University School of Art in New Haven, Connecticut.', ' Xavier University of Louisiana Department of Art has enlisted him as Chairman for over a decade, and, in addition, Bechet has served as Acting Chairman for the Department of Fine Arts & Philosophy at Southern University at New Orleans for many years.']], ['Reese Witherspoon', ['Laura Jeanne Reese Witherspoon ( ; born March 22, 1976) is an American actress, producer, and entrepreneur.', ' Born in New Orleans and raised in Tennessee, she began her career as a child actress, making her professional screen debut in \"The Man in the Moon\" (1991), for which she was nominated for a Young Artist Award.', ' Following breakout roles in \"\" (1992) and \"Jack the Bear\" (1993), she starred in the comedy-drama \"Pleasantville\" (1998), for which she won the Young Hollywood Award for Breakthrough Performance.', ' Her leading role of Tracy Flick in \"Election\" (1999) was nominated for a Golden Globe Award.']], ['Skip Bolen', ['Skip Bolen is a Southern photographer of musicians, architecture, lifestyle and the culture of New Orleans.', ' Born in Lafayette, Louisiana, he moved to New Orleans where he began his publishing career as a designer and art director.', ' After moving to New York City, he began working at \"House & Garden,\" renamed \"HG,\" as Senior Designer in January 1988 with Anna Wintour and Alexander Liberman at Condé Nast Publications.', ' Spending evenings in jazz clubs, he began photographing jazz musicians in New York City and often when he regularly returned to New Orleans.', ' After three years at Condé Nast Publications in New York City, he returned to New Orleans to pursue his jazz photography full-time.', ' In 1998, he moved to Los Angeles where he became art director of House of Blues for seven years while photographing at night and weekends.', ' He continued photographing jazz musicians and had his first major solo exhibition at the Jazz Bakery in Los Angeles on August 9, 2002.', ' On July 4, 2006, he returned to New Orleans to pursue photography full-time documenting the recovery and rebuilding of New Orleans since Hurricane Katrina, documenting the jazz scene, night-time photography and other photographic projects.']], ['Overnight Delivery', ['Overnight Delivery is a 1998 romantic comedy film directed by Jason Bloom.', ' It was rated PG-13 by the MPAA and released direct-to-video.', ' It featured Reese Witherspoon and Paul Rudd, prior to both becoming considerably bigger film stars.']], ['City of New Orleans (train)', ['The City of New Orleans is an Amtrak passenger train which operates on an overnight schedule between Chicago and New Orleans.', ' The train is a successor to the Illinois Central Railroad\\'s \"Panama Limited\".', ' The present name was revived in 1981.']], ['List of New Orleans Pelicans head coaches', ['The New Orleans Pelicans are an American professional basketball team based in New Orleans, Louisiana.', ' They play in the Southwest Division of the Western Conference in the National Basketball Association (NBA).', \" The team was born out of the original Hornets' relocation to New Orleans in 2002.\", ' The team has had three names since its inception; it was called the New Orleans Hornets (2002–2005; 2007–2013), the New Orleans / Oklahoma City Hornets (2005–2007), and the New Orleans Pelicans (2013–present).', ' The Pelicans have never been to the NBA Finals since its inception.', ' The team has played their home games at the New Orleans Arena since 2002.', ' The Pelicans are owned by Tom Benson, with Dell Demps as their general manager.']], ['Paul Soniat', ['Paul Soniat is the Director of the New Orleans City Park Botanical Gardens.', ' He was born at the old Touro Hospital in New Orleans.', ' His family has been in New Orleans since 1727 and he grew up and later lived on the street bearing his family name.', ' Paul is a self-taught piano player, and has released two CDs, absorbing the sights, sounds, and flavor of New Orleans.', ' His first CD, titled \"Born in New Orleans\", was released in April 2005.', \" His second CD was released after Hurricane Katrina hit the Gulf coast and the New Orleans' levees failed.\", ' This second CD is appropriately titled \"Below the Water Line\".']], ['Doreen Ketchens', ['Doreen Ketchens (born October 3, 1966 in New Orleans, Louisiana) is a jazz clarinetist, who performs Dixieland and Trad Jazz.', ' She is one of the first and few female bandleaders in New Orleans, and a musical educator.', \" She has performed at concert halls and music festivals, at U.S. Embassies and decades of weekly performances in Dixieland's tradition in the Royal Street Performing Arts Zone in the French Quarter of New Orleans with her band, Doreen's Jazz New Orleans.\", ' Ketchens has performed for four U.S. Presidents: Bill Clinton, George Bush Sr., Ronald Reagan and Jimmy Carter, and is widely considered one of the cultural ambassadors of New Orleans and of the traditional music']], ['Mimeo, Inc', ['Mimeo.com, Inc. is a privately held Print on demand and digital distribution document company.', ' It was the first to offer online printing and overnight delivery of complex documents and marketing materials.', ' The company refers to itself as a technology company that prints.', ' Customers utilize a proprietary online workflow connected to multiple print production, warehouse and distribution centers.', ' Customers include small, mid-sized and large companies.', ' The company was named after the Mimeograph.', ' Printing and distribution centers are located in Memphis, Tennessee, Newark, New Jersey and Huntingdon, Cambridgeshire, England.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.648\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7651825542992d0ec060a3', 'answer': 'Ateneo de Davao University', 'question': 'Which university has more schools, Ateneo de Davao University or University of Rochester?', 'supporting_facts': [['Ateneo de Davao University', 0], ['Ateneo de Davao University', 3], ['Ateneo de Davao University', 6], ['University of Rochester', 2]], 'context': [['Ateneo de Iloilo', ['The Ateneo de Iloilo – Santa Maria Catholic School (AdI–SMCS), (), is a private, Catholic, Chinese Filipino preparatory school run by the Philippine Province of the Society of Jesus in Iloilo City, Philippines.', ' Ateneo de Iloilo began in 1958 as a parochial school named Santa Maria Catholic School.', ' In 2004, the school was officially recognized as a Jesuit school separate from the parish and was renamed Ateneo de Iloilo - Santa Maria Catholic School.', ' It is the eighth Jesuit school in the Philippines to be named Ateneo.', ' Ateneo de Iloilo is a K-12 school and its curriculum includes a Chinese language program.']], ['R.J. Rizada', ['Ryan Joseph Ramos Rizada, better known as R.J. Rizada, (born October 5, 1982 in Davao) is a Filipino former professional basketball player who played in the Philippine Basketball Association.', ' He was the twelfth overall pick in the 2006 PBA Draft.', ' He played for the Ateneo de Davao Blue Knights for a year and was recruited by the Far Eastern University Tamaraws after he was scouted in the University games.']], ['Tanghalang Ateneo', ['Tanghalang Ateneo, the longest-running theater company of the Loyola Schools, Ateneo de Manila University, weaves into its work the theatrical traditions of the University’s sesquicentennial past: the devotion of the \"salon de actos\" at the \"Escuela Municipal\", the eloquence of the Ateneo Dramatics Guild, the \"joie de vivre\" of the Ateneo Players Theater, and the innovative spirit of the Ateneo Experimental Theater.', ' Like these companies, Tanghalang Ateneo uses theater to foster \"eloquentia\", \"sapientia\", and \"humanitas\" – the pillars of Jesuit pedagogy.', ' It sees itself as a theater company in the service of student formation, and by extension – given the Jesuit ideal of \"magis\" – a theater in service of the nation.']], ['Ateneo de Manila University', ['The Ateneo de Manila University (Filipino: \"Pamantasang Ateneo de Manila;\" Spanish:\" Universidad Ateneo de Manila\") is a private research university in Quezon City, Philippines.', ' Founded in 1859 by the Society of Jesus, the Ateneo is the third-oldest university in the Philippines.', ' Ateneo offers elementary and secondary education exclusively to male students (and has recently opened the Senior High School to girls).', ' The undergraduate and graduate programmes are coeducational and organized into four schools, collectively known as the Loyola Schools, which are located at its main campus at Loyola Heights.', ' Four professional schools occupy campuses in different parts of Metro Manila.']], ['Lex Talionis Fraternitas', ['Lex Talionis Fraternitas, Inc.', ' Sodalitas Ducum Futurorum is an exclusive fraternal organization of Filipino jurists, legal practitioners and law students founded on September 29, 1969 at the San Beda College of Law.', ' A chapter in the Ateneo de Davao University School of Law was established in 1974.', ' In 1983, the Securities and Exchange Commission granted the incorporation of the fraternity.']], ['Leoncio P. Deriada', ['He was born in Iloilo but spent most of his life in Davao.', ' He went to school at the Davao City High School and graduated in 1955.', ' He earned his BA English degree at the Ateneo de Davao University where he graduated cum laude in 1959.', ' He later received his MA in English from Xavier University in 1970 and went on to receive his PhD in English and Literature with a specialization in creative writing from Silliman University in 1981 where he later on served as professor and chairperson of the English Department.']], ['Ateneo de Tuguegarao', ['The Ateneo de Tuguegarao was a Catholic college in Tuguegarao, Cagayan, Philippines run by the Society of Jesus.', ' It was established in 1945 when the Jesuits took over the administration of the diocesan secondary school, \"Cagayan Valley Atheneum\".', ' The Jesuits renamed the school Ateneo de Tuguegarao after taking control.', ' Ateneo de Tuguegarao had a high school and college departments.', ' It was the fifth Ateneo that the Jesuits established in the Philippines.', ' In 1962, the school was closed when the Jesuits left Tuguegarao.']], ['Joel Tabora', ['Joel E. Tabora (born September 26, 1947, Manila, Philippines) is a Jesuit priest and the president of Ateneo de Davao University.', ' He demonstrated a commitment to \"whole person formation\", social justice and spirituality.']], ['Ateneo de Davao University', ['The Ateneo de Davao University is a private teaching, service and research university run by the Society of Jesus in Davao City in the Philippines.', ' It is also known by the acronym AdDU.', ' It was established in 1948, and is the seventh Ateneo opened by the Jesuits in the Philippines.', ' The university has five undergraduate schools, namely the School of Arts and Sciences, School of Business and Governance, School of Engineering and Architecture, School of Education and the School of Nursing.', ' The graduate programs are under these units as well.', ' The College of Law is a separate unit within the university.', ' The university also runs a grade school and high school.']], ['University of Rochester', ['The University of Rochester ( U of R or UR) frequently referred to simply as Rochester, is a private, nonsectarian, research university in Rochester, New York.', ' The university grants undergraduate and graduate degrees, including doctoral and professional degrees.', ' The university has six schools and various interdisciplinary programs.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.648\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8ec3205542995a26add506', 'answer': 'yes', 'question': 'Does Dashboard Confessional have more members than World Party?', 'supporting_facts': [['Dashboard Confessional', 0], ['World Party', 0]], 'context': [['MTV Unplugged 2.0', ['MTV Unplugged 2.0 is a live album released by Dashboard Confessional in 2002.', \" This CD/DVD package is the band's first live album.\", ' The band already recorded an acoustic instrument-based LP and is also the first non-Platinum selling artist to be on \"MTV Unplugged\".', ' After a few months, RIAA certified the album Platinum, indicating shipment of between 100,000 and 200,000 units, as it is considered a long-form video.', ' The album is the first one to have peaked at #1 on the Top Heatseekers chart and the Top Independent Albums chart.', ' The album peaked at #111 on the \"Billboard\" 200.', ' It is the only Dashboard Confessional LP album with a platinum certification.']], ['So Impossible EP', [\"So Impossible EP is Dashboard Confessional's second EP.\", ' It was released on December 18, 2001 through Vagrant Records.', ' The EP release was also widely acclaimed and received great scores in music review magazines like SPIN.', \" All four songs would later be performed in Dashboard Confessional's MTV Unplugged performance.\", ' The same performance was released as an album in 2002.']], ['Rooftops and Invitations', ['\"Rooftops and Invitations\" is a download-only single from the album \"Dusk and Summer\" by Dashboard Confessional.', ' The song was written by the lead singer of Dashboard Confessional, Chris Carrabba.', ' \"Rooftops and Invitations\" was released to radio on August 29, 2006.']], ['Vindicated (song)', ['\"Vindicated\" is a song by Dashboard Confessional released on the 2004 soundtrack for the film \"Spider-Man 2\" as well as on Dashboard Confessional\\'s 2006 album, \"Dusk and Summer\", as a bonus track on some pressings and on deluxe edition versions.', ' Played over the film\\'s end credits, \"Vindicated\" is the theme for the film.']], ['Dashboard Confessional', ['Dashboard Confessional is an American emo band from Boca Raton, Florida, led by singer-songwriter Chris Carrabba.', ' The name of the band is derived from the song \"The Sharp Hint of New Tears\" off their debut album, \"The Swiss Army Romance\".']], ['World Party', ['World Party is a British alternative rock band, which is essentially the solo project of its sole member, Karl Wallinger.', ' He started the band in 1986 in London after leaving The Waterboys.']], ['Mike Marsh (musician)', ['Mike Marsh (born August 13, 1974 in Miami, Florida) is the drummer for The Avett Brothers and formerly of Dashboard Confessional.', ' Mike was featured on all Dashboard Confessional\\'s albums, from \"The Places You Have Come To Fear The Most\" to \"Alter the Ending\".', ' He also played with them for MTV Unplugged.', ' In 2006, Marsh and Dashboard Confessional did an AOL Sessions recording, playing not only their songs, but a cover of \"In A Big Country\" by Big Country.', ' According to posts by Mike on Facebook, he officially joined The Avett Brothers as their drummer in early 2013.']], ['Jay Orpin', ['Jay Orpin (born April 29, 1976 in Stockholm, Sweden ) is a Swedish and Finnish songwriter and producer.', ' He also writes songs and produced for Backstreet Boys, NSYNC, Robyn, Ace of Base, Bon Jovi, 2gether and Britney Spears, and later he produced songs for Good Charlotte, Sum 41, Simple Plan, Hawthorne Heights, Red Jumpsuit Apparatus, My Chemical Romance, Tokio Hotel, Lindsay Lohan, Dashboard Confessional, All American Rejects, Fall Out Boy, AFI, Evanescence, Hollywood Undead, t.A.T.u., Yellowcard, Hannah Montana, Linkin Park, Bullet for My Valentine and Taking Back Sunday.', \" Most of his music is under the influence of today's modern pop punk, dance-punk, and emo music, but he does write some pop and hip-hop music as well.\", \" He also teamed up with Travis Barker in 2007, remixing popular hip-hop singles such as Throw Some D's, Party Like a Rockstar, and Crank That Soulja Boy.\", ' Even though he has written some of the works of some of the aforementioned artists, he is not one of the main producers of many of these artists and he chooses not to put his credit for his work.', ' An original song written by such composer to be cited is yet to be found, however some has claimed an abbreviation of his name in one of the albums.', ' In late 2008, he has decided to move in a small town near Oulu, Finland to raise his family and is a high school teacher and does accounting for bill paying in the summer to support the family, along with music production projects part-time, but he still continues to produce music today, but not as much as he used to.', ' He did come back to help write a few new songs, such as In My Head by Jason Derulo.', ' He has quit his part-time job in accounting to allow more time back in the music production business, to make a revival.']], ['Seville (band)', ['Seville is a rock group formed in the winter of 2001, by Mike Marsh of the Agency and Dan Bonebrake of The Vacant Andys, joined the now-well-known Dashboard Confessional.', ' They disbanded in 2003, with all members following other projects.']], [\"Don't Wait (Dashboard Confessional song)\", ['\"Don\\'t Wait\" is the first single from the album \"Dusk and Summer\" by Dashboard Confessional.', ' The song was written by the lead singer of Dashboard Confessional, Chris Carrabba.', \" It is about living for the day, that 'the moment is now'.\", ' \"Don\\'t Wait\" was released to radio on May 23, 2006.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.649\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae7c2dc5542994a481bbdd3', 'answer': 'North Dakota', 'question': 'Father Jean-Pierre Aulneau de le Touche was killed before he could go on an expedition to Mandan, which is located in which state? ', 'supporting_facts': [['Jean-Pierre Aulneau', 0], ['Mandan', 0]], 'context': [[\"Le Touquet – Côte d'Opale Airport\", ['Le Touquet - Côte d\\'Opale Airport (French: \"Aéroport de Le Touquet - Côte d\\'Opale\" ) (IATA: LTQ,\\xa0ICAO: LFAT) is 2.9 km east-southeast of Le Touquet, a commune of the Pas-de-Calais department in the Nord-Pas-de-Calais region of France.', \" It is on Côte d'Opale, the northern coast of France\"]], ['Louis de Lotbiniere-Harwood', ['Dr Louis de Lotbinière-Harwood (1866–1934) M.D., F.A.C.S., was a Canadian gynaecologist.', ' He was Dean of Medicine at Université de Montréal, the second campus of Université Laval.', ' He was President of the Medical Union of Canada, President of the Hôpital Notre-Dame and President of the Radium Institute, Paris.', ' His reputation as an educator and a surgeon extended throughout North America and Europe, recognised through his creation as an \"Officier de Le Légion d\\'honneur\" in France.', \" He has been referred to as the 'Father of Canadian Gynaecology'.\"]], ['Jean-Pierre Aulneau', ['Father Jean-Pierre Aulneau de la Touche, S.J. (21 April 1705 – 8 June 1736) was a Jesuit missionary priest who was briefly active in New France and killed before he could take part in his first major assignment which was to be an expedition to the Mandan.', ' He died near Fort St. Charles, on Lake of the Woods in an area now in Ontario, Canada and Minnesota, United States.', ' He was killed while traveling with Jean Baptiste de La Vérendrye, and is often referred to as \"Minnesota\\'s Forgotten Martyr.\"']], ['Le Maire Strait', ['The Le Maire Strait (\"Estrecho de le Maire\") (also the Straits Lemaire) is a sea passage between Isla de los Estados and the eastern extremity of the Argentine portion of Tierra del Fuego.']], ['Lament of Edward II', ['The \"Lament of Edward II\", \"En tenps de iver me survynt damage\" (sic), is traditionally credited to Edward II of England, and thought to have been written during his imprisonment shortly after he was deposed by his wife Isabella in January 1327.', ' Not all readers are convinced of the royal attribution of its authorship.', ' The poem, in fifteen stanzas, bears the heading \"De Le Roi Edward, le Fiz Roi Edward, Le Chanson Qe Il Fist Mesmes\" (\"Of the King Edward, son of King Edward, the Song that He Made himself\").', ' It was a \"chanson\", and was likely to be sung to an existing tune.', ' In each stanza two rhymes alternate, in approximately octosyllabic lines.', ' The text survives in a manuscript on vellum at Longleat, bound into a volume titled \"Tractatus varii Theologici saec.', ' XIII et XIV\" (76v and 77r), causing it to be overlooked; and in a manuscript in the Royal Library.', ' It was identified by Paul Studer and first published by him with a short literary introduction and an English translation in 1921.']], ['Mandan High School', ['Mandan High School is a public high school located in Mandan, North Dakota.', ' It is the only high school within the Mandan Public School system, serving grades 9–12.', ' In 2007 the Mandan Public School District renovated the high school.', ' Mandan is the 6th-largest school district in the state of North Dakota.', ' Mandan High School has an 88% graduation rate.', ' Enrollment for the 2009–2010 school year was 1,056 students.', ' The graduating class for the 2009–2010 school year was 229 students.', ' On April 1, 2010, Mandan High School was recognized for maintaining 100 years of continuous accreditation from the North Central Association Commission on Accreditation and School Improvement (NCA CASI).']], ['Mandan', ['The Mandan are a Native American tribe residing in North Dakota.', ' They are enrolled in the Three Affiliated Tribes of the Fort Berthold Reservation.', ' About half of the Mandan still reside in the area of the reservation; the rest reside around the United States and in Canada.']], ['Convent de Le Celle, Cortona', ['The Convent de Le Celle is a 13th-century Franciscan Convent located in Le Celle, just outside Cortona, region of Tuscany, Italy.', ' It is also referred to as the \"Convento delle Celle\" or \"Eremo Le Celle\".']], ['Christian Moueix', ['Christian Moueix (] ; born 1946) is a French winemaker and the president of the \"négociant\" house Établissements Jean-Pierre Moueix in Libourne, overseeing production in several estates in Saint-Émilion and Pomerol including Château Pétrus and Château Trotanoy.', ' He has managed the company since his father Jean-Pierre Moueix stepped down in 1978.']], ['Claude-Godefroy Coquart', ['Claude-Godefroy Coquart (February 2, 1706 – July 4, 1765) was a Jesuit priest who probably arrived in Quebec in 1739.', ' He was almost immediately assigned to accompany La Vérendrye to the western forts.', ' He was to replace Father Jean-Pierre Aulneau who had lost his life in the massacre on Lake of the Woods in 1736.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.650\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8b595855429949d91db563', 'answer': '1992', 'question': 'In what year was the band, whose bassist joined Arvas in 1996, formed?', 'supporting_facts': [['Arvas', 1], ['Gorgoroth', 1]], 'context': [['Herzog (band)', ['Herzog is an indie rock band from Cleveland, Ohio formed in 2010.', ' Their music has been described as \"subtlety-free rock,\" and has been compared to the Cloud Nothings, whose bassist, TJ Duke, formerly played in Herzog.', \" The band's frontman, Nick Tolar, graduated from Saint Ignatius High School in 2000.\", ' They became well-known when NPR chose the song \"Silence\" from their debut album Search as a song of the year in 2011.', ' Their third album, \"Boys\", was released on May 20, 2014.', \" Dan Price, the band's drummer, has said that its music resembles Weezer and the other '90's bands he and his bandmates grew up listening to.\", ' He has also said that \"Boys\" is the best representation of his band\\'s sound yet.']], ['Coldplay', ['Coldplay are a British rock band formed in 1996 by lead vocalist and keyboardist Chris Martin and lead guitarist Jonny Buckland at University College London (UCL).', ' After they formed under the name Pectoralz, Guy Berryman joined the group as bassist and they changed their name to Starfish.', ' Will Champion joined as drummer and backing vocalist, completing the lineup.', ' Creative director Phil Harvey is often referred to as the fifth member by the band.', ' The band renamed themselves \"Coldplay\" in 1998, before recording and releasing three EPs: \"Safety\" in 1998 and \"Brothers & Sisters\" and \"The Blue Room\" in 1999.', ' \"The Blue Room\" was their first release on a major label, after signing to Parlophone.']], ['Modest Mouse', ['Modest Mouse is an American rock band formed in 1992 in Issaquah, Washington (a suburb of Seattle), and currently based in Portland, Oregon.', ' The founding members are lead singer/guitarist Isaac Brock, drummer Jeremiah Green, and bassist Eric Judy.', ' Strongly influenced by groups Pavement, the Pixies, XTC, and Talking Heads, the band rehearsed, rearranged, and recorded demos for almost two years before finally signing with small-town indie label, K Records, and releasing numerous singles.', ' Since the band\\'s 1996 debut album, \"This Is a Long Drive for Someone with Nothing to Think About\", the group\\'s lineup has centered on Brock and Green.', ' Judy performed on every Modest Mouse album until his departure in 2012.', ' Guitarist Johnny Marr (formerly of the Smiths) joined the band in 2006, shortly following percussionist Joe Plummer (formerly of the Black Heart Procession) and multi-instrumentalist Tom Peloso, to work on the album \"We Were Dead Before the Ship Even Sank\".', ' Guitarist Jim Fairchild joined the band in 2009.', ' The band\\'s sixth album, \"Strangers to Ourselves\", was released on March 17, 2015.']], ['Geno Arce', [\"Geno Arce (born in Portland, Oregon), started playing bass in the clubs at age 16 opening for bands like Black 'n Blue and Fire Eye.\", ' Graduating from high school in 1982, he joined the navy and did his stint for his country.', ' After receiving an honorable discharge, he returned to Portland for a short while and then relocated to Phoenix Arizona.', ' While in Phoenix Geno performed on the local scene and in Los Angeles with the bands “Syngin, SX, and Box of Cherries, which later became “The Einsteins.”', ' In 1998 Geno joined forces with Ron Keel to form the “Roadhouse Rattlers” and began his journey into Southern Rock and Country Music.', ' In 1999 Ron Keel had to take a trip to Europe and Geno Joined Phoenix Arizona’s “Harry and the Gila Monsters\" furthering a career in country music that would allow him to share the stage with acts such as Brooks and Dunn, Reba McIntyre, Montgomery Gentry, Neal McCoy, and many others.', ' In 2000, Geno and Ron Keel were reunited, moved to Plain City Ohio where they formed the international southern rock band “IronHorse” consisting of Ron Keel vocals, Geno Arce Bass, Robert Marcelo Guitar, and Gaetano Nicolosi on drums.', ' During its five year lifetime IronHorse performed all throughout the United States opening for bands such as “The Outlaws and Ted Nugent” In 2008 Geno Joined the all original lineup of Keel consisting of Ron Keel, Marc Ferrari, Bryan Jay, Dwain Miller and replacing bassist Kenny Chaisson.', ' In June 2016 Geno was asked by Ron Keel to join him in another project called Badlands House Band.', ' This band is part of a larger project in Sioux Falls SD called Badlands Pawn.', ' and the brain child of Chuck Brennan.']], ['Mathien', ['Mathien is a Midwestern funk rock band named after lead singer and guitarist Chris Mathien.', ' Chris wrote, produced, and played all the parts on his first album \"Head, Heart & Hands\" (2007).', ' To form a live band Chris met with bassist Mike Schiff and drummer Aaron Bouslog and formed the group in Carbondale, Illinois, at Southern Illinois University.', ' They would later be joined by keyboardist George Jackson after relocating to Chicago, Illinois.', ' Lee England Jr. on violin was also added as a guest for the recording of the album \"Hello, Again\" (2009).', ' The band played a packed House of Blues in Chicago to kick off the album and tour.', ' With songs like \"Little Richard\", \"Dirt That I Do\",\"Goodbye\", \"Remember\" and \"We Don\\'t Need to Make Love, to Know That We\\'ve Got it\" the band had built a strong fan base and played shows on a 2009–2010 tour.', ' 2011 brought some changes with the band for the recording of \"The Night I was an Alpha Male\" (2011).', ' George Jackson was out and Peter Wilkins joined the band on keyboards.', ' They kicked off with an album release party at House of Blues again and toured in the Midwest.', ' The band also broke into college radio playlists across the country; radio support came from North Central College WONC-FM in Naperville IL and Findlay College WLFC-FM in Findley, Ohio, and many others.', ' The title track received heavy air play plus songs like \"Jamie\\'s Son\", \"Betaman\", \"Rub It In\", \"The Hold\" and the huge crowd favorite \"Lettuce Head\".', ' 2012 the band changed drummers and welcomed Omar Jahwar to the band, Mathien began a midwest tour during 2012 building a fan base at every show they played.', \" Later that year Chris Mathien was on WONC-FM's Local Chaos radio show and said the band will be recording a new album for release in 2013.\"]], ['Patrick Bruders', ['Patrick Bruders is an American musician, best known as the current bassist for heavy metal supergroup Down, the former bassist of the blackened death metal band Goatwhore from 1997 to 2004, and the former bassist for the seminal sludge metal band Crowbar from 2005 until 2013.', ' In 2008 he joined Eyehategod side project Outlaw Order and began live bass duties for the band, but has since parted ways with the group.', ' He began touring as live bassist with heavy metal supergroup Down in early 2011, replacing former bassist Rex Brown, before being added as a permanent member, performing bass duties live and in the studio.', ' Bruders is also a member of the New Orleans-based crust punk band Gasmiasma, Austin-based country band Pure Luck, and joined legendary doom metal band Saint Vitus for their performance at the Hammer of Doom music festival in Würzburg, Germany.']], ['Gorgoroth', ['Gorgoroth is a Norwegian black metal band based in Bergen.', ' It was formed in 1992 by guitarist Infernus, who is also the only original member remaining, and the band have since released nine studio albums.', ' Gorgoroth are a Satanic band and have drawn controversy due to some of their concerts, which have featured impaled sheep heads and mock crucifixions.', ' The band is named after the dead plateau of darkness in the land of Mordor from J. R. R. Tolkien\\'s fantasy novel \"The Lord of the Rings\".']], ['L7 discography', ['The discography of the rock band L7.', ' L7 was formed by Donita Sparks and Suzi Gardner on shared electric guitar and shared vocals in 1985.', ' A year prior, Gardner had performed backing vocals on the Black Flag song \"Slip It In\".', ' The band also consisted of bassist Jennifer Finch and drummer Roy Koutsky (who was in the band for a year and a half).', ' After their debut the band recruited Demetra Plakas as their permanent drummer.', ' This line up continued through their albums, Smell the Magic in 1990 with Subpop records, with signing to the record label Slash the band had recorded three albums, Bricks Are Heavy in 1992, and Hungry for Stink in 1994.', ' By the time 1996 came around Finch had wanted to leave.', ' While the band was preparing to record their next album.', ' Greta Brink had taken over bass for the new album, .', ' Which was released in 1997.', ' Gail Greenwood took over as their bassist for their final album, Slap-Happy which was produced by their own record company, Wax Tadpole records.', ' After this the band started to break up when Greenwood left the band.', ' Janis Tanaka had taken over bass.', ' However the band ended up going into an hiatus, in 2001.']], ['Joe Raposo (bassist)', ['Joe Raposo (born 1970), of Portuguese descent, is the bassist for the seminal California punk rock band Lagwagon and qa engineer lead at Zynga.', ' He also played bass for a period of time for The Real Mckenzies and Mad Caddies.', ' Additionally, Raposo plays bass for the San Francisco-based fusion group King City, with fellow Lagwagon member Chris Rest as well as filling in on bass for several live shows with the Dwarves.', ' Raposo began his career in 1987 at the age of seventeen by joining California hardcore punk band Rich Kids on LSD as their new bassist and remained with RKL until their hiatus in 1996.', ' He began playing shows with RKL again in 2003 (after they had re-formed a year prior) until their current hiatus after the death of lead singer Jason Sears.', ' Raposo joined Lagwagon in 2010, replacing original bassist Jesse Buglione.', ' Raposo also plays bass in the cover band Uke-Hunt.']], ['Arvas', ['Arvas, formerly known as Örth, is a Norwegian black metal band formed in Bergen in 1993 as a one-man project by multi musician V-Rex.', ' In Summer 1996, Borknagar drummer Grim and Gorgoroth bassist Ares joined the band.', ' They recorded an album in Winter 1996, titled \"Nocturno Inferno\", but it was never released.', \" After Grim committed suicide in 1999, V-Rex decided to keep on going as a one-man project, but changed the band's name.\", ' He recorded two demos, \"Countless Souls at Dawn\" and \"I Am Thy Grief\", and a split album with Hordagaard titled \"Dawn of Satan/Uncle Satan\", which was released by Azermedoth Records.', ' The band\\'s first official full-length album \"Blessed from Below – Ad Sathanas Noctum\" was released in 2010 by the band themselves.', ' Their second album \"Into The Realm Of The Occult\" was released in November 2013 by Italian label ATMF Records, to be followed in March 2015 by \"Black Satanic Mysticism\", via Aeternitas Tenebrarum Music Foundation.', ' Arvas has shared stages with bands like Throne Of Cartasis, Urgehal, Dauden, 1349, Mongo Ninja, Nocturnal Breed.', \" In March 2013 Arvas toured the east European countries supporting Deicide on their 'End Of The World Tour'.\", ' The band\\'s fourth full-length, \"Black Path\", was released on 24 March 2017 by Mighty Music.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.650\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adfb698554299603e418385', 'answer': 'Tonde Burin', 'question': 'Who created the manga series originally serialized from 1994-1995with a superheroine?', 'supporting_facts': [['Tonde Burin', 0], ['Tonde Burin', 1], ['Superhero', 0]], 'context': [[\"Menacing Dog's\", ['Menacing Dog\\'s (Japanese: キョウハクDOG\\'s , Hepburn: Kyōhaku Dog\\'s ) is a Japanese manga series written and illustrated by Shaa, the same illustrator of the \"Nogizaka Haruka no Himitsu\" light novel series.', ' The manga was originally serialized in MediaWorks\\' \"Dengeki Teioh\" magazine, but after the magazine became defunct, it began serialization in \"Dengeki G\\'s Festival!', ' Comic\", renamed Menacing Dog\\'s: Another Secret (キョウハクDOG\\'s -Another Secret- , Kyōhaku Dog\\'s -Another Secret- ) .', ' The manga was serialized in the two magazines between the November 2005 and January 2012 issues.', ' A total of four \"tankōbon\" volumes were published under the Dengeki Comics imprint.', ' Infinity Studios licensed \"Menacing Dog\\'s\" in North America, and \"Menacing Dog\\'s: Another Secret\"\\'s chapters are digitally serialized in English on Kadokawa\\'s Comic Walker website.']], ['Stray Little Devil', ['Stray Little Devil (ストレイ リトル デビル , Sutorei Ritoru Debiru ) is a Japanese manga series written and illustrated by Kotaro Mori.', ' The manga was originally serialized in Dengeki Comic Gao!', ', and was later published into five bound volumes by MediaWorks from January 2005 to March 2007.', ' DrMaster Publications Inc. licensed the manga series for English-language publication in North America and released the five volumes between June 2006 and November 2007.', ' The story is a comedic fantasy, detailing the life of a girl named Pam Akumachi, who is unintentionally drawn into the \"Spirit World\".', \" Her only hope of returning home is a mysterious individual named Remy, who promises to tell Pam the way back if she becomes a full-fledged devil by passing through the devils' educational system.\"]], ['Superhero', ['A superhero (sometimes rendered super-hero or super hero) is a type of heroic stock character who possesses supernatural or superhuman powers and who is dedicated to fighting crime, protecting the public, and usually battling supervillains.', ' A female superhero is sometimes called a superheroine (also rendered super-heroine or super heroine), although the word superhero is commonly used for females also.', ' Superhero fiction is the genre of fiction that is centered on such characters, especially in American comic books since the 1930s.']], ['Toward the Terra', ['Toward the Terra (Japanese: <ruby><rb>地球</rb><rp>(</rp><rt>テラ</rt><rp>)</rp></ruby>へ… , Hepburn: Tera e..', '. )', ' is a Japanese science fiction manga series by Keiko Takemiya.', ' It was originally serialized in Asahi Sonorama\\'s \"Gekkan Manga Shōnen\" magazine, between January 1977 and May 1980.', ' In 1978, it won the very first Seiun Award for manga, and in 1980 also won the Shogakukan Manga Award for shōnen/shōjo manga (along with Takemiya\\'s \"Kaze to Ki no Uta\").']], ['List of Shadow Star chapters', ['Shadow Star (Japanese: Narutaru (なるたる ) ) is a Japanese manga series created by Mohiro Kitoh, originally serialized in Kodansha\\'s seinen magazine \"Afternoon\".', ' In the United States, it was licensed by Dark Horse and serialized in \"Super Manga Blast!', '\".']], ['Nono-chan', ['Nono-chan (ののちゃん ) is a yonkoma manga series begun in 1991 by Hisaichi Ishii originally serialized as My Neighbors the Yamadas (となりのやまだ君 , Tonari no Yamada-kun ) in the \"Asahi Shimbun\" in Japan.', ' When the series first began, it was generally focused on all of the members of the Yamada family.', ' As the series progressed, the daughter (Nonoko, or \"Nono-chan\") became the most popular character among readers and more of the strips focused on her and her point of view.', ' In 1997, the series title was changed to reflect this change of focus.', ' The \"Asahi Shimbun\" continues to feature this manga series as of October 2007.']], ['Shadow Star', ['Shadow Star, known in Japan as Narutaru (Japanese: なるたる ) , is a Japanese manga series created by Mohiro Kitoh, originally serialized in Kodansha\\'s seinen magazine \"Afternoon\".', ' The Japanese name is an abbreviation of \"Mukuro Naru Hoshi, Tama Taru Ko\" (骸なる星 珠たる子 ) , which roughly translates to \"Corpse of a Star; A Precious Child\".', ' In the United States, it was licensed by Dark Horse and serialized in \"Super Manga Blast!', '\".']], ['Chirality (manga)', ['Chirality (キラリティー , Kiraritī ) is a 4-volume yuri manga series written and illustrated by author Satoshi Urushihara.', ' The manga was originally serialized in \"Comic NORA\" in 1995, and later published in three bound volumes, which was re-released into two bound volumes in 2003.', ' In 1997 \"Chirality\" was licensed for released in North America by Central Park Media.', ' It was originally published as 18 issues between March 1997 and August 1998, as well as being released into four bound volumes from 1997 to 2000.', ' The art was also flipped so that it would read left to right which was not an uncommon practice for manga released in Western Hemisphere at the time.']], ['Touch (manga)', ['Touch (Japanese: タッチ , Hepburn: Tatchi ) is a Japanese high school baseball manga by Mitsuru Adachi.', ' It was originally serialized in the weekly manga magazine \"Shōnen Sunday\" from 1981–1986, and sold over 100 million copies, making it one of the best-selling manga series.', ' The manga was also adapted into a 101-episode anime television series – which was one of the highest-rated anime television series ever, three theatrical anime movies which summarized the TV series, two anime television specials which take place after the events in the TV series, a live-action TV drama special, and a live-action movie released in 2005.', ' \"Touch\" was one of the winners of the 1983 Shogakukan Manga Award for shōnen or shōjo manga, along with Adachi\\'s \"Miyuki\".']], ['Tonde Burin', ['Tonde Burin (とんでぶーりん , Tonde Būrin ) is a Superhero magical girl manga series created by Taeko Ikeda.', ' It is originally serialized in Shogakukan\\'s Shōjo magazine \"Ciao\" from August 1994 to September 1995, collecting into 3 Tankobon Volumes.', ' An anime series based on the manga was created by Nippon Animation and was broadcast on all MBS stations in Japan from September 3, 1994 through August 26, 1995.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.651\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab94bc2554299743d22eacf', 'answer': '300', 'question': 'Bytham Castle is a castle in the civil parish of how many houses?', 'supporting_facts': [['Bytham Castle', 0], ['Castle Bytham', 0]], 'context': [['Little Bytham', ['Little Bytham is a village and civil parish in the South Kesteven district of Lincolnshire, England.', ' The population of the civil parish at the 2011 census was 384.', ' It lies on the B1176 road, 4 mi south from Corby Glen and 6 mi north from Stamford .']], ['Castle Bytham', ['Castle Bytham is a village and civil parish of around 300 houses in South Kesteven, Lincolnshire, England.', ' The population was measured at 768 in 317 households at the 2011 census.']], ['Castle Ashby', ['Castle Ashby is the name of a civil parish, an estate village and an English country house in rural Northamptonshire.', ' Historically the village was set up to service the needs of Castle Ashby House, the seat of the Marquess of Northampton.', ' The village has one small pub-hotel, The Falcon.', \" At the time of the 2011 census, the parish's population (including Chadstone) was 111 people.\", ' The village contains many houses rebuilt from the 1860s onwards.', ' These include work by the architect E.F. Law of Northampton, whose work can also be seen nearby at Horton Church.', ' The castle is the result of a licence obtained in 1306, for Walter Langton, Bishop of Coventry, to castellate his mansion in the village of Ashby.']], ['Grendon, Northamptonshire', ['Grendon is a small village and civil parish in rural Northamptonshire, England on the borders of Bedfordshire and Buckinghamshire.', ' Many houses are made of the local limestone and various older thatched houses still survive.', ' The name of the village means \"green hill\" and today the village remains centred on the hill.', ' As with Earls Barton, the village was owned by Judith, the niece of William the Conqueror.']], ['Newbiggin, Ainstable', ['Newbiggin is a small hamlet in Cumbria, England Cumrew beck flows north-west through Newbiggin eventually joining the Eden close to Armathwaite.', ' The village contains many houses of a traditional design, a historic chapel (now a private home) and several large acreage farms.', ' On the fells around the village there are traces of the old mines that used to operate in the area, as well as the skeletons of Lime kiln.', ' A track from the village leads up to new water river, which can be followed north to Castle Carrock.', ' There is a pub, The Blue Bell Inn.']], ['Goose Creek Historic District', ['The Goose Creek Historic District is a rural landscape in the Goose Creek valley of Loudoun County, Virginia.', ' The district covers about 10000 acre south of Hamilton and Purcellville and includes the village of Lincoln.', ' The majority of the district is farmland, with areas of forest along Hogback Mountain.', ' The area was settled by Quakers in the mid-18th century, represented by simple houses and the Goose Creek Meetinghouse Complex in Lincoln, separately listed on the National Register of Historic Places.', ' About 270 buildings lie within the district.', ' The district includes 44 stone buildings, reflecting the popularity of this material in the 18th and 19th centuries in this area.', ' Many houses have outbuildings and barns built in a manner complementary to the dwellings.', ' By the mid-19th century, materials turned to brick, with the Glebe of Shelburne Parish an NRHP-listed example of a brick Federal style house, as well as the Israel Janney House.']], ['Senhora da Hora', ['Senhora da Hora (] ) is a former civil parish in the municipality of Matosinhos in the Greater Porto area, Portugal.', ' In 2013, the parish merged into the new parish São Mamede de Infesta e Senhora da Hora.', ' It was promoted from town (\"vila\") to city (\"cidade\") status on 12 June 2009.', ' It lies just north of the Porto city limits and is densely populated.', ' It is a major suburban habitational area with many houses and some commerce.']], ['Bytham Castle', ['Bytham Castle was a castle in the village of Castle Bytham in Lincolnshire (grid reference [ SK992186] .)']], ['Church Minshull', ['Church Minshull is a village and civil parish in the unitary authority of Cheshire East and the ceremonial county of Cheshire, England.', ' The village is located approximately 5 mi north west of Crewe and to the west of the River Weaver and the Shropshire Union Canal.', ' The principal road through Church Minshull is the B5074 between Nantwich (6 miles to the south) and Winsford (4 miles to the north).', ' The modern village centre is a designated conservation area which contains many houses of Tudor style architecture.', ' A large area in the east of the parish falls within the Weaver Valley Area of Special County Value.']], ['East Garston', ['East Garston is a village and civil parish on the River Lambourn, about 5.5 mi north of Hungerford in West Berkshire.', ' The river flows through the village, dividing many houses from the main road, so that each has a bridge over the river to the front door.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.652\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a78caa955429970f5fffd83', 'answer': 'the Etruscan civilization', 'question': 'Which civilization instigated a war with Rome and significantly influenced the Latin language?', 'supporting_facts': [['Lars Tolumnius', 0], ['Etruscan language', 0], ['Etruscan language', 1]], 'context': [['Etruscan language', ['The Etruscan language ( ) was the spoken and written language of the Etruscan civilization, in Italy, in the ancient region of Etruria (modern Tuscany plus western Umbria and northern Latium) and in parts of Campania, Lombardy, Veneto, and Emilia-Romagna, where the Etruscans were later displaced by Gauls.', ' Etruscan influenced Latin, but was eventually completely superseded by it.', ' The Etruscans left around 13,000 inscriptions which have been found so far, only a small minority of which are of significant length, some bilingual inscriptions with texts also in Latin, Greek, or Phoenician, and a few dozen loanwords, such as the name Roma (from Etruscan \"Ruma\"), but Etruscan\\'s influence was significant.']], ['Saga of Western Man', ['Saga of Western Man is a historically themed anthology series television series that aired on ABC Television from 1963 to 1969.', ' Each episode focused on a particular year, person, or incident that producer John H. Secondari felt significantly influenced the progress of Western civilization.']], ['Livius Andronicus', ['Lucius Livius Andronicus (c. 284 – c. 205 BC) was a Greco-Roman dramatist and epic poet of the Old Latin period.', ' He began as an educator in the service of a noble family at Rome by translating Greek works into Latin, including Homer\\'s \"Odyssey\".', ' They were meant at first as educational devices in the school he founded.', ' He wrote works for the stage—both tragedies and comedies—which are regarded as the first dramatic works written in the Latin language of ancient Rome.', ' His comedies were based on Greek New Comedy and featured characters in Greek costume.', ' Thus, the Romans referred to this new genre by the term comoedia palliata (fabula palliata).', ' The Roman biographer Suetonius later coined the term \"half-Greek\" of Livius and Ennius (referring to their genre, not their ethnic backgrounds).', \" The genre was imitated by the next dramatists to follow in Andronicus' footsteps and on that account he is regarded as the father of Roman drama and of Latin literature in general; that is, he was the first man of letters to write in Latin.\", ' Varro, Cicero, and Horace, all men of letters during the subsequent Classical Latin period, considered Livius Andronicus to have been the originator of Latin literature.', ' He is the earliest Roman poet whose name is known.']], ['English language', ['English is a West Germanic language that was first spoken in early medieval England and is now a global \"lingua franca\".', ' Named after the Angles, one of the Germanic tribes that migrated to England, it ultimately derives its name from the Anglia (Angeln) peninsula in the Baltic Sea.', ' It is closely related to the Frisian languages, but its vocabulary has been significantly influenced by other Germanic languages, particularly Norse (a North Germanic language), as well as by Latin and Romance languages, particularly French.']], ['Aquitani', ['The Aquitanians (Latin: Aquitani) were a people living in what is now southern Aquitaine and southwestern Midi-Pyrénées, France, called Gallia Aquitania by the Romans in the region between the Pyrenees, the Atlantic ocean, and the Garonne, present-day southwestern France.', ' They were an ancient non-Indo-European population that lived in the northern slopes of the Pyrenees.', ' They spoke the Aquitanian language, related to Old Basque.', ' Classical authors such as Julius Caesar and Strabo clearly distinguish them from the other peoples of Gaul and Hispania (the Iberian Peninsula).', ' With the process of Romanization, in the centuries of Roman Empire, they adopted the Latin Language (Vulgar Latin) and Roman civilization.', ' Their old language, the Aquitanian language, was the substrate for the Gascon language (one of the Romance languages) spoken in Gascony.']], ['Lars Tolumnius', ['Lars Tolumnius (Etruscan: Larth Tulumnes, d. 437 BC), was the most famous king of the wealthy Etruscan city-state of Veii, roughly ten miles northwest of Rome, best remembered for instigating a war with Rome that ended in a decisive Roman victory.']], ['Classical Latin', ['Classical Latin is the modern term used to describe the form of the Latin language recognized as standard by writers of the late Roman Republic and the Roman Empire.', ' In some later periods, it was regarded as \"good\" Latin, with later versions being viewed as debased or corrupt.', ' The word \"Latin\" is now taken by default as meaning \"Classical Latin\", so that, for example, modern Latin textbooks describe Classical Latin.', ' Marcus Tullius Cicero and his contemporaries of the late republic, while using \"lingua latina\" and \"sermo latinus\" to mean the Latin language as opposed to Greek or other languages, and \"sermo vulgaris\" or \"sermo vulgi\" to refer to the vernacular, referred to the speech they valued most and in which they wrote as \"latinitas\", \"Latinity\", with the implication of good.', ' Sometimes it was called \"sermo familiaris\", \"speech of the good families\", \"sermo urbanus\", \"speech of the city\" or rarely \"sermo nobilis\", \"noble speech\".', ' But besides \"latinitas\", it was mainly called \"latine\" (adverb), \"in good Latin\", or \"latinius\" (comparative degree of the adverb), \"good Latin\".']], ['Jean Bayet', ['Jean Bayet (12 November 1892 – 5 December 1969) was a French Latinist.', ' A Professor of Latin Language and Literature at the Sorbonne, he was Director-General of Education in 1944 and Director of the École française de Rome from 1952 to 1960.', ' In 1948 he was elected a member of the Académie des inscriptions et belles-lettres.', ' A specialist of Latin literature and Religion in ancient Rome, Jean Bayet, through his works and the theses he directed, played a decisive role in the development of a French school of history of the Roman religion, particularly active in the second half of the twentieth century.']], ['Contemporary Latin', ['Contemporary Latin is the form of the Latin language used from the end of the 19th century through to the present.', ' Various kinds of contemporary Latin can be distinguished.', ' On the one hand there is its survival in areas such as taxonomy as the result of the widespread presence of the language in the New Latin era.', ' This is usually found in the form of mere words or phrases used in the general context of other languages.', ' On the other hand, there is the use of Latin as a language in its own right as a full-fledged means of expression.', ' Living or Spoken Latin, being the most specific development of Latin in the contemporary context, is the primary subject of this article.']], ['Renaissance music', ['Renaissance music is vocal and instrumental music written and performed in Europe during the Renaissance era.', ' Consensus among music historians\\xa0– with notable dissent – has been to start the era around 1400, with the end of the medieval era, and to close it around 1600, with the beginning of the Baroque period, therefore commencing the musical Renaissance about a hundred years after the beginning of the Renaissance as it is understood in other disciplines.', ' As in the other arts, the music of the period was significantly influenced by the developments which define the Early Modern period: the rise of humanistic thought; the recovery of the literary and artistic heritage of Ancient Greece and Ancient Rome; increased innovation and discovery; the growth of commercial enterprises; the rise of a bourgeois class; and the Protestant Reformation.', ' From this changing society emerged a common, unifying musical language, in particular the polyphonic style (this means music with multiple, independent melody lines performed simultaneously) of the Franco-Flemish school, whose greatest master was Josquin des Prez.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.653\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae688065542996d980e7beb', 'answer': '122,067', 'question': 'What was the population at the 2010 census of the city which, along with Clayton, is covered by the Mount Diablo Unified School District?', 'supporting_facts': [['Mount Diablo Unified School District', 4], ['Concord, California', 1]], 'context': [['Lammersville Joint Unified School District', ['Lammersville Joint Unified School district (LJUSD) (formerly Lammersville Elementary School District) is a pre-kindergarten through twelfth grade unified school district in Mountain House, California, which serves the area west of Tracy and parts of Alameda County.', ' The district was created when majority of voters in the Lammersville and Mountain House area passed a measure to separate from Tracy Unified School District in a special election on June 8, 2010.', ' It became an independent school district on July 1, 2011.']], ['San Ramon Valley Unified School District', ['San Ramon Valley Unified School District is a public school district in Contra Costa County, California.', ' The San Ramon Valley Unified School District encompasses the communities of Alamo, Blackhawk, Danville, Diablo, and San Ramon (including the new Dougherty Valley communities) as well as a small portion of the cities of Walnut Creek and Pleasanton.', ' The district operates 35 schools serving more than 30,000 students in Kindergarten through Grade 12.']], ['Concord, California', ['Concord ( ) is the largest city in Contra Costa County, California.', ' At the 2010 census, the city had a population of 122,067 making it the 8th largest city in the San Francisco Bay Area.', ' Founded in 1869 as the community of Todos Santos by Salvio Pacheco, the name was changed to Concord within months.', ' The city is a major regional suburban East Bay center within the San Francisco Bay Area, and is 29 mi east of San Francisco.']], ['KVHS', ['KVHS (90.5 FM, \"90.5 The Edge\") is a non-profit high school radio station playing an Active Rock music format.', ' It is licensed to Clayton Valley High School under the jurisdiction of the Mount Diablo Unified School District and broadcasts from the campus of Clayton Valley Charter High School, Concord, California, USA.', ' The signal reaches the counties of Contra Costa, Solano, Napa, San Joaquin, West Sacramento and Yolo, and KVHS primarily serves the Diablo Valley area.']], ['Ygnacio Valley High School', ['Ygnacio Valley High School (YVHS) is a public secondary school located in Concord, California, United States.', ' It draws students from Concord as well as from the neighboring communities of Walnut Creek and Pleasant Hill.', ' The school opened in 1962, and its first senior class graduated in 1964.', ' Originally conceived as a temporary facility, the school currently carries an enrollment of over 1,500 total students for grades 9 through 12.', ' When the nearby Northgate High School opened in 1974, YVHS lost approximately half its student body at the time.', ' The school is part of the Mount Diablo Unified School District.']], ['Concord High School (California)', ['Concord High School is located at 4200 Concord Blvd. in Concord, California, United States, near El Dorado Middle School and Westwood Elementary.', ' As of 2014, the current principal is Rianne Pfaltzgraff.', ' The school educates nearly 1700 students, and it continues to grow.', ' It is one of the six high schools in the Mount Diablo Unified School District.', ' Concord High School was constructed in 1966 and currently provides 144373 sqft in permanent structure, including about 70 classrooms, a library, and other structures.']], ['Northgate High School (Walnut Creek, California)', ['Northgate High School (NHS) is a public high school located in the suburban Northgate neighborhood of Walnut Creek, California, United States.', ' The most recent of five high schools in the Mount Diablo Unified School District, the school was built in 1974, and is home to approximately 1,600 students from Walnut Creek and Concord, California, grades 9-12.', ' Its name derives from its location at the north entrance of Mount Diablo State Park.']], ['Hesperia Unified School District', ['Hesperia Unified School District is a school district in San Bernardino County, California.', ' Hesperia Unified School District serves the City of Hesperia and adjacent areas in the High Desert of San Bernardino County and covers 161 square miles.', ' The Hesperia Unified School District provides public education services for kindergarten through senior high school students.', ' It includes 3 comprehensive high schools, 2 continuation high schools, 3 middle schools, 12 elementary schools, 3 choice schools, 2 alternative schools, 1 adult education school, and 5 charter schools.']], ['Rancho Monte del Diablo', ['Rancho Monte del Diablo was a 17921 acre Mexican land grant in present-day Contra Costa County, California given in 1834 by Governor José Figueroa to Salvio Pacheco.', ' The name \"Monte del Diablo\" means \"thicket of the devil\" in Spanish.', ' The name was later incorrectly translated as Mount Diablo.', ' The grant covered the area from the Walnut Creek channel east to the hills, and generally from the Mount Diablo foothills north along Lime Ridge to Avon on the Carquinez Strait of the Sacramento River, and included present day Concord and parts of Pleasant Hill.', ' Pacheco and Clayton are outside of the Rancho Monte del Diablo grant.']], ['Mount Diablo Unified School District', ['Mount Diablo Unified School District (MDUSD) is a public school district in Contra Costa County, California.', ' It currently operates 29 elementary schools, 10 middle schools, and five high schools, with 7 alternative school programs and an adult education program.', ' MDUSD is one of the largest school districts in the state of California, with over 56 school sites and a budget of approximately $270,000,000.', ' The district has over 36,000 K-12 students, over 20,000 adult education students, and over 3,500 employees, including over 2,000 certificated educators.', ' The district covers 150 sqmi , including the cities of Concord and Clayton; as well as most of Pleasant Hill and portions of Walnut Creek, Pittsburg, Lafayette, and Martinez; and unincorporated areas, including Pacheco, Clyde, and Bay Point.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.653\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abaccd1554299660624161b', 'answer': 'McComb, Mississippi', 'question': \" Curious is a women's fragrance by a singer born in what city and state?\", 'supporting_facts': [['Curious (fragrance)', 0], ['Britney Spears', 0], ['Britney Spears', 1]], 'context': [['Nude by Rihanna', ['Nude by Rihanna is the third fragrance for women by Barbadian singer Rihanna.', ' The fragrance was released on November 23, 2012 (Black Friday), under the perfume line, Parlux Fragrances.']], ['Rebelle (fragrance)', ['Rebelle is the second fragrance for women by Barbadian singer Rihanna.', ' The fragrance was released in spring 2012 under the perfume line Parlux Fragrances.', ' The fragrance was made available for purchase on Belk.com, on February 15, 2012.', ' in the United States.', \" It was also available in a special pack at Macy's for a limited time.\", ' The pack included the singer\\'s previous fragrance, Reb\\'l Fleur, her sixth studio album, \"Talk That Talk\" and Rebelle.']], ['Eau de Gaga', ['Eau de Gaga is the second fragrance created by American singer Lady Gaga.', ' The announcement and details of the fragrance were announced on her Haus Laboratories website.', \" The fragrance's notes include white violet, lime, and leather, and it is marketed as being suitable for both men and women.\"]], ['Ambition (fragrance)', [\"Ambition is the third women's fragrance created by American pop/R&B singer, songwriter Jordin Sparks alongside CPL Aromas & Preferred Fragrance, endorsed by Jordin Sparks.\", ' The product was released exclusively to Bon-Ton Department Stores nationwide on November 8, 2012 in store and online.', ' Ambition... was Preceded by two additional releases.', ' her first fragrance \"Because of You...\" and her second fragrance \"Fascinate\".', ' Each scent was followed with its own Eau De Parfum release and multiple gift sets.']], [\"Reb'l Fleur\", [\"Reb'l Fleur is the first fragrance for women endorsed by Barbadian singer Rihanna.\", ' The fragrance was promoted with the accompanying tagline \"Bad feels so good\".', \" Reb'l Fleur was released in the United States on January 25, 2011, and in the United Kingdom on August 19, 2011.\", ' The product was positively received upon its release, and it became highly successful.']], ['Curious (fragrance)', ['Curious is a women\\'s fragrance by Britney Spears for Elizabeth Arden, and is the first perfume to be endorsed by Britney Spears, preceding \"Fantasy.\"', ' Spears reportedly earned $52 million from the endorsement.']], ['Britney Spears', ['Britney Jean Spears (born December 2, 1981) is an American singer, dancer, and actress.', ' Born in McComb, Mississippi, and raised in Kentwood, Louisiana, she performed acting roles in stage productions and television shows as a child before signing with Jive Records in 1997.', ' Spears\\'s first and second studio albums, \"...Baby One More Time\" (1999) and \"Oops!', '... I Did It Again\" (2000), became international successes, with the former becoming the best-selling album by a teenage solo artist.', ' Title tracks \"...Baby One More Time\" and \"Oops!', '... I Did It Again\" broke international sales records.', ' In 2001, Spears released her self-titled third studio album, \"Britney\", and played the starring role in the film \"Crossroads\" (2002).', ' She assumed creative control of her fourth studio album, \"In the Zone\" (2003), which yielded the worldwide success of the single \"Toxic\".']], ['Fantasy (fragrance)', [\"Fantasy is a women's fragrance and fragrance line by Britney Spears and Elizabeth Arden.\", ' Fantasy, the perfume, is the second perfume to be endorsed by Britney Spears.', ' It was released in the US on September 15, 2005, following the successful previous perfume from Spears, \"Curious\", which made over $30 million in sales in the first three months of its launch.', ' The Fantasy fragrance line has currently fourteen perfumes: Fantasy, Midnight Fantasy, Hidden Fantasy, Circus Fantasy, Fantasy Twist, Island Fantasy, Fantasy Anniversary Edition, Fantasy The Nice Remix, Fantasy The Naughty Remix, Fantasy Stage Edition, Rocker Femme Fantasy, Fantasy Renner Edition, Fantasy Intimate Edition, and Maui Fantasy.']], ['Rogue by Rihanna', ['Rogue is the fourth fragrance released by Barbadian singer Rihanna.', \" The fragrance was released in two installments; firstly a women's fragrance and finally a men's cologne.\", \" The women's fragrance was officially released on September 4, 2013 under the perfume line, Parlux Fragrances and was made available for purchase on Rihanna's official perfume website and also at Macy's, Dillards and Belk stores.\"]], ['Radiance (fragrance)', [\"Radiance is a women's fragrance and ninth released by Britney Spears for Elizabeth Arden, September 2010.\", ' Spears first revealed the fragrance in her Twitter account, later revealing through the same social network an advertisement, which features the singer in a long silver dress, that was later donated to raise funds for The Matthew Van Daff Special Needs Trust.', ' The top notes of \"Radiance\" are wild berries and soft, dewy petals, mixed with tuberose, jasmine, orange blossom and iris.', ' With the tagline \"Choose your own destiny\", the fragrance is available as 50 and 100 ml EDP.', ' \"Radiance\" was also promoted in Spears\\' music videos for \"Hold It Against Me\" (2011) and \"Criminal\" (2011).']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.654\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5addda1b5542992200553b55', 'answer': 'Polish', 'question': 'Wilm Hosenfeld assisted the pianist and composer of what nationality survive during 1944?', 'supporting_facts': [['Wilm Hosenfeld', 1], ['Władysław Szpilman', 0]], 'context': [['David Ezra Okonşar', ['David Ezra Okonşar (] ; born 20 October 1961, Istanbul) is a Turkish-Belgian (double nationality) pianist, composer, conductor, writer and educator.', ' He was previously known as \"Mehmet Okonşar\".']], ['Władysław Szpilman', ['Władysław Szpilman (] ; 5 December 19116 July 2000) was a Polish pianist and classical composer of Jewish descent.', ' Szpilman is widely known as the central figure in the 2002 Roman Polanski film \"The Pianist\", which was based on Szpilman\\'s autobiographical account of how he survived the German occupation of Warsaw and the Holocaust.']], ['The Pianist (memoir)', ['The Pianist is a memoir by the Polish-Jewish pianist and composer Władysław Szpilman in which he describes his life in Warsaw in occupied Poland during World War II.', ' After being forced with his family to live in the Warsaw ghetto, Szpilman manages to avoid deportation to the Treblinka extermination camp, and from his hiding places around the city witnesses the Warsaw ghetto uprising in 1943 and the Warsaw uprising (the rebellion by the Polish resistance) the following year.', ' He survives in the ruined city with the help of friends and strangers, including Wilm Hosenfeld, a German army captain who admires his piano playing.']], ['Bebu Silvetti', ['Juan Fernando Silvetti Adorno (27 March 1944 – 5 July 2003), professionally known as Bebu Silvetti or simply Silvetti, was an Argentine-born Mexican pianist, composer, conductor, arranger, and record producer.', ' In the 1970s he moved to Mexico and got the nationality.', ' Popularly known for the 1977 instrumental disco hit, \"Lluvia De Primavera\" (\"Spring Rain\" in English), the album was produced in Mexico and for the 1980 modern instrumental mariachi album.', ' Silvetti was also a successful, Grammy-winning producer for a wide variety of Latin and international music performers.', ' He was the father of the actress Anna Silvetti.', ' Silvetti also worked in the music of successful films and telenovelas in Mexico.']], ['Wilm Hosenfeld', ['Wilhelm Adalbert Hosenfeld (] ; 2 May 1895 – 13 August 1952), originally a school teacher, was a German Army officer who by the end of the Second World War had risen to the rank of \"Hauptmann\" (Captain).', ' He helped to hide or rescue several Polish people, including Jews, in Nazi-occupied Poland, and helped Polish-Jewish pianist and composer Władysław Szpilman to survive, hidden, in the ruins of Warsaw during the last months of 1944, an act which was portrayed in the 2002 film The Pianist.', ' He was taken prisoner by the Red Army and died in Soviet captivity seven years later.']], ['Jimmy Bowien', ['Jimmy Bowien (born February 5, 1933, in Koenigsberg/Prussia) is a German record producer, songwriter and composer.', ' He started playing the piano at the age of 5 discovering his love for music early on in life.', ' Bowien moved to Hamburg-Germany to study opera singing (Baritone) under the guidance of the distinguished vocal coaches Wilm Schmieding and Harry Voges.', ' After finishing his studies and becoming a Baritone-singer, he applied for a job position at the record label Polydor in Hamburg and over many years and a large body of work became one of the most recognized music producers in Germany.']], ['Thomas Kretschmann', ['Thomas Kretschmann (] ; born 8 September 1962) is a German actor.', ' He played Lieutenant Hans von Witzland in the 1993 film \"Stalingrad\", Hauptmann Peter Kahn in the 2013 film \"Stalingrad\", Hauptmann Wilm Hosenfeld in \"The Pianist\", Hermann Fegelein in \"Downfall\", Major Otto Remer in the 2008 film \"Valkyrie\", and Captain Engelhorn in the 2005 remake of \"King Kong\", and voiced Professor Z in \"Cars 2\".', ' He appeared as Baron Strucker in Marvel Studios\\' \"\" and \"\".']], ['Krzysztof Książek', ['Krzysztof Książek \\xa0\\xa0 was born on 5 August 1992 in Cracow, is a Polish classical pianist from Krakow, he is a winner of highest prizes at national and international piano competitions.', ' His nomation is Polityka Passport Award for Classical Music.', ' Since 2005, he has trained with Stefan Wojtas, first at music secondary school, now as a student of the Feliks Nowowiejski Academy of Music in Bydgoszcz.', ' He has won prizes in Polish and international competitions, including the Ricard Vines in Leida, Spain, the Lviv international competitions in Ukraine, the \"Città di Avezzano\" in Italy (2011, First Prize) and the Polish Frédéric Chopin Piano Competition in Warsaw (2011, Third Prize; 2015, Second Prize ex aequo), XV International Pianist Ricard Vines (Lleida, Spain 2009), The International Chopin Piano Competition (Lviv, Ukraine 2010), VI Concorso Internazionale Pianistico \"Citta di Avezzano\" (Avezzano, He participated in the 67th International Chopin Festival in Duszniki-Zdrój, the 18th International Festival of Young Winners of Silesian Music Contests, 5th National Promotion Festival \"August Talents\" He is a scholarship recipient of the \"Sapere Auso\" Malopolska Foundation, the Prime Minister, the Ministry of Culture and National Heritage, and the Pro Musica Bona Foundation.', ' His nationality is Polish.']], ['Malinchism', ['Malinchism (Spanish: \"malinchismo\" ) or malinchist (Spanish: \"malinchista\" ) is a form of attraction that the foreigner has in the popular imagination, causing individuals to lose the spirit of nationality by moving to the other side, a particular case of cultural cringe.', \" It is derived from the name of Hernán Cortés's Nahua advisor La Malinche, referring to a deep-rooted Mexican inferiority complex or self-hatred for the preference for all things foreign to the point of self-destruction.\", \" La Malinche is used as a symbol for being supposed to have assisted the Spaniards in destroying Native Americans' way of life, values, norms and culture and exploit the Native American peoples.\"]], ['Rustem Hayroudinoff', ['Rustem Hayroudinoff (Russian: Рустем Афзалович Хайрутдинов ) is a Russian concert pianist.', ' Tatar by nationality, he was born in Kazan, Russian Federation (Republic of Tatarstan).', ' His father, Afzal Hayroudinoff is a Professor of Cello at the Kazan State Conservatory.', ' He is a brother of Halida Hayrutdinova, also acclaimed concert pianist.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.654\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7b3d565542992d025e67b6', 'answer': 'The Andes', 'question': 'In what mountain range that extends through multiple countries is there a mountain that lies southeast of Jacabamba?', 'supporting_facts': [['Auqui', 2], ['Andes', 0], ['Andes', 3]], 'context': [['Awqa Urqu', ['Awqa Urqu (Quechua \"awqa\" enemy, \"urqu\" mountain, \"enemy mountain\", Hispanicized spelling \"Auca Orjo\") is a 4982 m mountain in the Chunta mountain range in the Andes of Peru.', ' It is situated in the Huancavelica Region, Castrovirreyna Province, Aurahuá District.', ' Awqa Urqu lies southeast of Wichinka Lake.']], ['Qaflankuh Mountain Range', ['Qaflankuh or Qaflankuh Mountain Range (Persian: رشته کوه\\u200cها\\u200cی قافلانکوه / Reshteh Kuh-hā-ye Qāflānkuh) is a mountain range that is located about 20 kilometres north of the city of Zanjan in Zanjan Province, Iran.', ' With an average width of almost 20 kilometres and a length of about 100 kilometres, this mountain range is stretched in a northwest-southeast direction, beginning from northwest of Zanjan, passing north of Zanjan and ending southeast of Zanjan.', ' The Qezel Owzan River flows in a valley that separates Qaflankuh Mountains on the south from Talish Mountains on the north.', ' With an elevation of 2971 metres, Mount Sendan Dagh is the highest mountain of the range which is located in the southeast section of this mountain range.', ' There are some mineral hot springs containing sulphur in different parts of this range.', ' Geologically, Qaflankuh Mountain Range was formed during the Tertiary volcanism and plutonism and is made mainly of the Eocene volcanic rocks with very small sections of intrusive rocks in the northern and southern parts of the range.']], ['Yana Urqu (Jaqhichuwa)', ['Yana Urqu (Quechua \"yana\" black, \"urqu\" mountain, \"black mountain\", hispanicized spelling \"Yanaorjo\") is a mountain in the Willkanuta mountain range in the Andes of Peru, about 4800 m high.', ' It is situated in the Cusco Region, Quispicanchi Province, Marcapata District.', ' Yana Urqu lies southeast of the mountain Jaqhichuwa.']], ['Qillqata (Condesuyos-La Unión)', ['Qillqata (Aymara \"qillqaña\" to write, \"-ta\" a suffix to indicate the participle, \"written\" or \"something written\", hispanicized \"Quelcata\") is a mountain in the Wansu mountain range in the Andes of Peru, about 5000 m high.', ' It is located in the Condesuyos Province, Cayarani District, and in the La Unión Province, Puyca District.', \" Qillqata lies southeast of the mountain Pilluni, southwest of the mountain Janq'u Q'awa and east of the mountain Ch'uwañuma.\"]], ['Yanajirca (Huallanca)', ['Yanajirca or Yana Hirka (Quechua \"yana\" black, Ancash Quechua \"hirka\" mountain, \"black mountain\", also spelled \"Yanajirca\") is a mountain in the Andes of Peru which reaches an altitude of approximately 4600 m .', ' It is located in the Ancash Region, Bolognesi Province, Huallanca District.', ' Yana Hirka lies southeast of the Wallanka mountain range.']], ['Elkhead Mountains', ['The Elkhead Mountains are a mountain range in Colorado.', ' The mountain range is considered to be low altitude within Colorado as the mountains are under 11000 ft .', ' Located within Routt and Moffat counties, the mountain range is far from metropolitan areas and has few lakes and streams, so it attracts few visitors.', ' The mountain range is a volcanic range and all of the peaks were formed by volcanic action.', ' The mountain range extends approximately 16 mi east to west and 10 mi north to south, and its center is located at , approximately 20 mi northeast of Craig and north of Hayden, Colorado 13 mi south of the Wyoming border.', ' Almost all of the peaks within the Elkhead Mountains are a part of Routt National Forest.', ' Significant peaks are: Bears Ears, Sugar Loaf, Saddle Mountain, Black Mountain, Pilot Knob, and Meaden Peak.']], ['Andes', ['The Andes or Andean Mountains (Spanish: \"Cordillera de los Andes\" ) are the longest continental mountain range in the world.', ' They form a continuous highland along the western edge of South America.', ' This range is about 7000 km long, about 200 to wide (widest between 18° south and 20° south latitude), and of an average height of about 4000 m .', ' The Andes extend from north to south through seven South American countries: Venezuela, Colombia, Ecuador, Peru, Bolivia, Argentina and Chile.']], [\"Llamayuq Q'asa\", ['Llamayuq Q\\'asa (Quechua \"llama\" llama, \"-yuq\" a suffix, \"Llamayuq\" an archaeological site, \"q\\'asa\" mountain pass, \"Llamayuq pass\" or \"mountain pass with llamas\", also spelled \"Llamayojcasa, Llamayoqccasa\"), also known as Qucha Quyllur (\"Qochaqollur\", \"lake star\"), is a mountain in the eastern extensions of the Urupampa mountain range in the Andes of Peru, about 4600 m high.', ' It is located in the Cusco Region, Calca Province, Calca District.', \" It lies southeast of Q'irayuq.\", ' This is where the archaeological site of Llamayuq is situated.']], ['Auqui', ['Awki (Quechua for prince; a mythical figure of the Andean culture; grandfather, hispanicized spelling \"Auqui\") is a mountain in the Cordillera Blanca in the Andes of Peru, about 4800 m high.', ' It is situated in the Ancash Region, Huari Province, Huari District.', ' Awki lies southeast of Jacabamba.']], ['Yanaqucha (Carabaya)', ['Yanaqucha (Quechua \"yana\" black, \"qucha\" lake, \"black lake\", hispanicized spelling \"Yanacocha\") is a mountain at a small lake of the same name in the Willkanuta mountain range in the Andes of Peru, about 5000 m high.', ' The mountain is located in the Puno Region, Carabaya Province, on the border of the districts Corani and Ollachea.', ' It lies southeast of the mountain Rit\\'i Wasi, northeast of the mountain T\\'ika Pallana and east of the mountain Llusk\\'a Rit\\'i and the lake Khichu Suytuqucha \"(Quicho Suytococha)\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.655\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a83b1e75542990548d0b220', 'answer': 'screenwriter', 'question': 'Worker: What professional title to both Christopher Nolan and Paul Schrader boast?', 'supporting_facts': [['Christopher Nolan', 0], ['Paul Schrader', 0]], 'context': [['Hardcore (1979 film)', ['Hardcore is a 1979 American crime drama film written and directed by Paul Schrader and starring George C. Scott, Peter Boyle and Season Hubley.', ' The story concerns a father searching for his daughter, who has vanished only to appear in a pornographic film.', ' Writer-director Schrader had previously written the screenplay for Martin Scorsese\\'s \"Taxi Driver\", and both films share a theme of exploring an unseen subculture.']], ['Paul Schrader', ['Paul Joseph Schrader (born July 22, 1946) is an American screenwriter, film director, and film critic.', ' Schrader wrote or co-wrote screenplays for four Martin Scorsese films: \"Taxi Driver\" (1976), \"Raging Bull\" (1980), \"The Last Temptation of Christ\" (1988), and \"Bringing Out the Dead\" (1999).', ' Schrader has also directed 18 feature films, including his directing debut crime drama, \"Blue Collar\" (co-written with his brother, Leonard Schrader), the crime drama \"Hardcore\" (a loosely autobiographical film also written by Schrader), his 1982 remake of the horror classic \"Cat People\", the crime drama \"American Gigolo\" (1980), the biographical drama \"\" (1985), the cult film \"Light Sleeper\" (1992), the drama \"Affliction\" (1997), the biographical film \"Auto Focus\" (2002), and the erotic dramatic thriller \"The Canyons\" (2013).']], ['Christopher Nolan', ['Christopher Edward Nolan ( ; born 30 July 1970) is an English-American film director, producer, and screenwriter.', ' He is one of the highest-grossing directors in history, and among the most successful and acclaimed filmmakers of the 21st century.']], ['The Yakuza', ['The Yakuza is a 1974 Japanese-American neo-noir gangster film directed by Sydney Pollack, written by Leonard Schrader, Paul Schrader, and Robert Towne.', \" The film is about a man (Robert Mitchum) who returns to Japan after several years away in order to rescue his friend's kidnapped daughter.\", ' Following a lackluster initial release, the film has since gained a cult following.']], ['Obsession (1976 film)', ['Obsession is a 1976 psychological thriller/mystery film directed by Brian De Palma, starring Cliff Robertson, Geneviève Bujold, John Lithgow, and Stocker Fontelieu.', ' The screenplay was by Paul Schrader, from a story by De Palma and Schrader.', \" Bernard Herrmann provided the film's soundtrack prior to his death in 1975.\", ' The story is about a New Orleans businessman who is haunted by guilt following the death of his wife and daughter during a kidnapping-rescue attempt.', ' Years after the tragedy, he meets and falls in love with a young woman who is the exact look-alike of his long dead wife.']], ['Old Boyfriends', ['Old Boyfriends is a 1979 American drama film directed by Joan Tewkesbury and written by Paul Schrader and Leonard Schrader.', ' The film stars Talia Shire, Richard Jordan, Keith Carradine, John Belushi, John Houseman and Buck Henry.', ' The film was released on April 13, 1979, by Embassy Pictures.']], ['The Walker', ['The Walker is a 2007 American-British drama film written and directed by Paul Schrader.', ' It is an independent production and is the latest installment in Schrader\\'s \"night workers\" series of films, starting with \"Taxi Driver\" in 1976, followed by \"American Gigolo\" in 1980 and \"Light Sleeper\" in 1992.']], ['Blue Collar (film)', ['Blue Collar is a 1978 American crime drama film directed by Paul Schrader, in his directorial debut.', ' It was written by Schrader and his brother Leonard, and stars Richard Pryor, Harvey Keitel and Yaphet Kotto.']], ['Dying of the Light (film)', ['Dying of the Light is a 2014 American psychological thriller film written and directed by Paul Schrader and starring Nicolas Cage, Anton Yelchin and Irène Jacob about a government agent who must track down and kill a terrorist before he loses his full memory from a disease.', ' It was released theatrically and through VOD formats by Lionsgate on December 5, 2014.', ' The film received extremely negative reviews, with controversy surrounding the heavy tampering and reediting of the footage by the studio, who denied Schrader final-cut privilege and led him and principal members of the cast to disown the released version and campaign against it.']], ['Leonard Schrader', ['Leonard Schrader (November 30, 1943 – November 2, 2006) was an American screenwriter and director, most notable for his ability to write Japanese language films and for his many collaborations with his brother, Paul Schrader.', ' He earned an Academy Award Nomination for the screenplay he wrote for the film \"Kiss of the Spider Woman\".']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.656\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a734bb65542994cef4bc50f', 'answer': 'Vincent Anthony Guaraldi', 'question': 'Fantasy Records is know for recordings of comedian Lenny Bruce, Creedence Clearwater Revival, and what American jazz pianist who composed the music for the animated television adaptations of the \"Peanuts\" comic strip?', 'supporting_facts': [['Fantasy Records', 1], ['Vince Guaraldi', 0]], 'context': [['Creedence Clearwater Revival (album)', ['Creedence Clearwater Revival is the debut studio album by the American rock band Creedence Clearwater Revival, released in 1968.']], ['Creedence Clearwater Revival Covers the Classics', ['Creedence Clearwater Revival Covers the Classics is a compilation album by Creedence Clearwater Revival.', ' Released in 2009, the album contains cover versions of songs as recorded by the band.']], ['Fantasy Records', ['Fantasy Records is an American record company and label founded by brothers Max and Sol Weiss in 1949.', ' The early years of the company were dedicated to issuing recordings by jazz pianist Dave Brubeck, who was also one of its investors, but the label is known more for its recordings of comedian Lenny Bruce, jazz musician Vince Guaraldi, and the rock band Creedence Clearwater Revival.']], ['Creedence Clearwater Revival: Box Set', ['Creedence Clearwater Revival: Box Set is a box set by Creedence Clearwater Revival, released in 2001.', ' It contains all of their complete studio albums, two complete live albums, and material recorded by the band under their previous names \"The Golliwogs\" and \"The Blue Velvets\".', ' In November 2013 the box set was reissued with different artwork.', ' This box set spans their career, and contains all of their complete materials.']], ['Creedence Clearwater Revisited', ['Creedence Clearwater Revisited is an American rock band formed in 1995 by two former members of Creedence Clearwater Revival – a much more well-known band with a very similar name.', ' It is known for playing the music that was originally made famous by Creedence Clearwater Revival.', ' The two common band members are Stu Cook (bass) and Doug \"Cosmo\" Clifford (drums).']], ['John Fogerty (album)', ['John Fogerty is the second solo studio album by former Creedence Clearwater Revival vocalist/guitarist John Fogerty, released in 1975.', ' It was released by Asylum Records in the United States and Fantasy Records internationally.', ' As with the Creedence Clearwater Revival records, the album consists of a mix of originals and cover songs.', ' Although the album is eponymously titled, Fogerty himself refers to it as \"Old Shep\"; Shep was the name of his dog, who appears on the cover with him.']], ['Vince Guaraldi', ['Vincent Anthony Guaraldi (July 17, 1928 – February 6, 1976), born Vincent Anthony Dellaglio, was an American jazz pianist noted for his innovative compositions and arrangements and for composing music for animated television adaptations of the \"Peanuts\" comic strip, as well as his performances on piano as a member of Cal Tjader\\'s late 50s ensemble and his own solo career which included the radio hit \"Cast Your Fate to the Wind\".']], ['The Best of Creedence Clearwater Revival', ['The Best of Creedence Clearwater Revival is a compilation album by American rock band Creedence Clearwater Revival, released in 1977.', ' The album features all the tracks of Chronicle which was released a year earlier in 1976, as well as \"Good Golly Miss Molly\", \"Born on the Bayou\", \"Cotton Fields\", \"Hello Mary Lou\", \"The Midnight Special\", \"Walk on the Water\", and \"Night Time Is the Right Time\" which would appear on the 1986 release \"Chronicle, Vol.', ' 2\", and \"Bootleg\" which does not appear on either volume.']], ['The Long Road Home', ['The Long Road Home: The Ultimate John Fogerty/Creedence Collection is a compilation album by American roots rock singer-songwriter John Fogerty, released on November 1, 2005, by Fantasy Records.', \" It compiles songs from Fogerty's solo career and his band Creedence Clearwater Revival.\", \" The title refers to Fogerty's return to Fantasy Records, after a lengthy stint with Warner Bros.\", ' Records and a brief stint with DreamWorks Records.']], ['Doug Clifford', ['Douglas \"Cosmo\" Clifford (born April 24, 1945 in Palo Alto, California) is an American drummer, best known as a founding member of Creedence Clearwater Revival.', ' After the group dissolved in the early 1970s, Clifford released a solo album and later joined CCR bassist Stu Cook in the Don Harrison Band.', ' In 1995, Clifford and Cook formed the band Creedence Clearwater Revisited, performing live versions of Creedence Clearwater Revival songs.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.657\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae762835542997b22f6a711', 'answer': 'tip of the Baja California', 'question': 'Were was the Mexican state after which there is Villa Unión, Sinaloa located? ', 'supporting_facts': [['Villa Unión, Sinaloa', 0], ['Mazatlán', 2]], 'context': [['Mazatlán', ['Mazatlán (] ) is a city in the Mexican state of Sinaloa.', ' The city serves as the municipal seat for the surrounding \"municipio\", known as the Mazatlán Municipality.', ' It is located at on the Pacific coast, across from the southernmost tip of the Baja California Peninsula.']], ['Villa Unión, Sinaloa', ['Villa Unión is the second largest town in the municipality of Mazatlán, after the port of Mazatlán.', ' It is located twenty kilometers south of the city on the banks of the Presidio River.']], ['Mitre Department', ['Mitre Department is a department of Argentina in Santiago del Estero Province.', ' The capital city of the department is Villa Unión.']], ['Poanas Municipality', ['Poanas is one of the 39 municipalities of Durango, in northwestern Mexico.', ' The municipal seat lies at Villa Unión.', ' The municipality covers an area of 1841\\xa0km².']], ['Mexican Federal Highway 40', ['Mexican Federal Highway 40, also called the \"Carretera Interoceánica\" (Interoceanic Highway), is a road beginning at Reynosa, Tamaulipas, just west of the Port of Brownsville, Texas, and ending at Mexican Federal Highway 15 in Villa Unión, Sinaloa, near Mazatlán and the Pacific coast.', ' It is called Interoceanic as, once finished, the cities of Matamoros, Tamaulipas, on the Gulf of Mexico and Mazatlán on the Pacific Ocean will be linked.']], ['Villa Unión', ['Villa Unión is a city in northwestern Argentina and the main settlement of Departamento Coronel Felipe Varela with a population of 12,263.']], ['Estado de Occidente', ['Estado de Occidente (also known as Sonora y Sinaloa) was a Mexican state established in 1824.', ' The constitution was drafted in that year and the government was initially established with its capital at El Fuerte, Sinaloa.', ' The first governor was Juan Miguel Riesgo.', ' The state consisted of modern Sonora and Sinaloa, and also modern Arizona more or less south of the Gila River (although in much of this area the Yaqui, Pima, Apaches, and other native inhabitants did not recognize the authority of the state),']], ['Villa Unión, Santiago del Estero', ['Villa Unión is a municipality and village in Santiago del Estero Province in Argentina.']], ['Villa Unión, Coahuila', ['Villa Unión is a city and seat of the municipality of Villa Unión, in the north-eastern Mexican state of Coahuila.']], ['Operation Sinaloa', ['Operation Sinaloa or Operation Culiacan - Navolato (Spanish: Operacion Sinaloa/Operacion Conjunto Sinaloa) is an ongoing Anti-drug trafficking operation in the Mexican state of Sinaloa by the Federal Police and the Mexican Armed Forces.', ' Its main objective is to cripple all cartel organizations such as the Sinaloa Cartel, Beltrán-Leyva Cartel and Los Zetas that operate in that state.', \" The Military was deployed in response to the murder of Mexico's Federal Police commissioner Édgar Eusebio Millán Gómez.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.658\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab8337a55429919ba4e225f', 'answer': 'yes', 'question': 'Are the movies \"Monsters, Inc.\" and \"Mary Poppins\" both by the same company?', 'supporting_facts': [['Monsters, Inc.', 0], ['Mary Poppins (film)', 0]], 'context': [['Mary Poppins (character)', ['Mary Poppins is a fictional character and the eponymous protagonist of P. L. Travers\\' \"Mary Poppins\" books and all of their adaptations.', ' A magical English nanny, she blows in on the East Wind and arrives at the Banks home at Number Seventeen Cherry Tree Lane, London, where she is given charge of the Banks children and teaches them valuable lessons with a magical touch.', ' Travers gives Poppins the accent and vocabulary of a real London nanny: cockney base notes overlaid with a strangled gentility.']], ['Mary Poppins (film)', ['Mary Poppins is a 1964 American musical-fantasy film directed by Robert Stevenson and produced by Walt Disney, with songs written and composed by the Sherman Brothers.', ' The screenplay is by Bill Walsh and Don DaGradi, loosely based on P. L. Travers\\' book series \"Mary Poppins\".', \" The film, which combines live-action and animation, stars Julie Andrews in the role of Mary Poppins who visits a dysfunctional family in London and employs her unique brand of lifestyle to improve the family's dynamic.\", ' Dick Van Dyke, David Tomlinson, and Glynis Johns are featured in supporting roles.', ' The film was shot entirely at the Walt Disney Studios in Burbank, California using painted London background scenes.']], ['Monsters, Inc.', ['Monsters, Inc. is a 2001 American computer-animated comedy film produced by Pixar Animation Studios and distributed by Walt Disney Pictures.', ' Featuring the voices of John Goodman, Billy Crystal, Steve Buscemi, James Coburn, and Jennifer Tilly, the film was directed by Pete Docter in his directorial debut, and executive produced by John Lasseter and Andrew Stanton.', ' The film centers on two monsters employed at the titular energy-producing factory Monsters, Inc. — top scarer James P. \"Sulley\" Sullivan and his one-eyed partner and best friend Mike Wazowski.', \" In the film, employees at Monsters, Inc. generate their city's power by scaring children, but they themselves are afraid that the children are toxic to them, and when one child enters the factory, Sulley and Mike must return her home before it is too late.\"]], ['Mary Poppins, Goodbye', ['Mary Poppins, Goodbye (Russian: Мэри Поппинс, до свидания!', ' ; translit.', '\\xa0\"Meri Poppins, do svidaniya\") is a Soviet 1983 two-part musical miniseries (part 1 \"Lady Perfection\", part 2 \"Week ends on Wednesday\"), directed by Leonid Kvinikhidze.', ' It is loosely based on Mary Poppins stories by P. L. Travers.', ' The TV series were ordered by the Gosteleradio of USSR and produced by Mosfilm.', ' The official television premiere was on January 8, 1984.']], ['Mary Poppins (song)', ['\"Mary Poppins\" is a song from the 2015 stage musical \"Love Birds\" with music and lyrics by Robert J. Sherman.', ' It is sung by \"The Original Quack Pack\", a penguin barbershop quartet who resemble the penguins from the 1964 Walt Disney motion picture, \"Mary Poppins\".', ' In dialogue leading up to the song, the penguins explain that while they did know the same nanny, (Mary Poppins) they are not the same penguins as in the movie.', ' The song expresses their longing for the magical nanny of literary fame.']], ['Chim Chim Cher-ee', ['\"Chim Chim Cher-ee\" is a song from \"Mary Poppins\", the 1964 musical motion picture.', ' It was originally sung by Dick Van Dyke and Julie Andrews, and also is featured in the Cameron Mackintosh/Disney \"Mary Poppins\" musical.', ' The song can be heard in the \"Mary Poppins\" scene of The Great Movie Ride at Disney\\'s Hollywood Studios and during the \"Mary Poppins\" segment of \"\" at Disneyland.']], ['I Love to Laugh', ['\"I Love to Laugh\", also called \"We Love to Laugh\", is a song from Walt Disney\\'s film \"Mary Poppins\".', ' It was composed by Richard M. Sherman and Robert B. Sherman.', ' The song is sung in the film by \"Uncle Albert\" (Ed Wynn), and \"Bert\" (Dick Van Dyke) as they levitate uncontrollably toward the ceiling, eventually joined by Mary Poppins (Julie Andrews) herself.', ' The premise of the scene, that laughter and happiness cause Uncle Albert (and like-minded visitors) to float into the air, can be seen as a metaphor for the way laughter can \"lighten\" a mood.', \" (Compare Peter Pan's flight power, which is also powered by happy thoughts.)\", ' Conversely, thinking of something sad literally brings Albert and his visitors \"down to earth\" again.', \" The song states a case strongly in favor of laughter, even if Mary Poppins appears to disapprove of Uncle Albert's behavior, especially since it not only complicates the task of getting Albert down, but the infectious mood sends Bert and the Banks children into the air as well.\"]], ['Mary Poppins Opens the Door', ['Mary Poppins Opens the Door is a British children\\'s fantasy novel by the Australian-British writer P.L. Travers, the third book and last novel in the \"Mary Poppins\" series that features the magical English nanny Mary Poppins.', ' It was published in 1943 by Harcourt, Brace & World, Inc and illustrated by Mary Shepard and Agnes Sims.']], ['Mary Poppins Returns', ['Mary Poppins Returns (also known as Mary Poppins 2) is an upcoming American musical fantasy film directed by Rob Marshall and written by David Magee.', ' It is the sequel to the 1964 film \"Mary Poppins\".', ' The film stars Emily Blunt, Lin-Manuel Miranda, Ben Whishaw, Emily Mortimer, Pixie Davies, Joel Dawson, Nathanael Saleh, Julie Walters, Colin Firth, Dick Van Dyke and Meryl Streep.', ' Set 25 years after the 1964 film, it will feature Mary Poppins, the former nanny of Jane and Michael Banks, re-visiting them after a family tragedy.', ' The film is scheduled for release on December 25, 2018, giving it one of the longest gaps between film sequels in history.']], ['Step in Time', ['\"Step In Time\" is a song and dance number from Walt Disney\\'s 1964 film \"Mary Poppins\", and it is composed by Richard M. Sherman and Robert B. Sherman.', ' The choreography for this song was provided by Marc Breaux and Dee Dee Wood.', ' It is sung by Bert, the chimney sweep (Dick Van Dyke) and the other chimney sweeps on the rooftops of London.', ' In the first part of the song, the lines he says in the verses are \"Kick your knees up\", \"\\'Round the chimney\", \"Flap like a birdie\", \"Up on the railing\", \"Over the rooftops\" and \"Link your elbows\" followed by an interlude.', \" The interlude continues with Bert, Mary Poppins, Michael, Jane and all the chimney sweepers dancing around the rooftops and as Admiral Boom looks at them with the telescope, he thinks that they're Hottentots, so he orders Mr. Binnacle to make them scram with colorful fireworks.\", ' In the second part, as all the chimney sweepers get in the house of George Banks, Mrs. Brill walks into the living room looking at them and screams, \"They\\'re at it again!\"', ' and she runs away trying to strike one of the chimney sweepers with a frying pan.', ' As Jane, Michael, Mary Poppins and Bert get in the same place, Ellen runs around the dining room with an \"OW!\"', ' and the chimney sweepers flip her.', ' The other phrases in the rest of the musical number are \"Votes for women,\" \"It\\'s the master,\" and \"What\\'s all this?\"']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.658\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5add1c485542992c1e3a253d', 'answer': 'wineries', 'question': 'What is a popular industry in the neighborhood of Willow Vale, New South Wales?', 'supporting_facts': [['Willow Vale, New South Wales (Wingecarribee)', 0], ['Willow Vale, New South Wales (Wingecarribee)', 1], ['Mittagong', 4]], 'context': [['Balaclava, New South Wales', ['Balaclava is a Northern Village of the Southern Highlands of New South Wales, Australia in Wingecarribee Shire.', ' It is 1\\xa0km north-east of Mittagong.', \" The village includes a service station, real estate, pre-school, nursery, doctor's surgery and antiques store.\", ' It is located in Wingecarribee Shire and is often considered part of Braemar along with its neighbour Willow Vale.', ' At the 2016 census , Balaclava had a population of 496.']], ['Minister for Trade and Industry (New South Wales)', ['The New South Wales Minister for Trade and Industry is a minister in the Government of New South Wales who has responsibilities for sponsoring and supporting trade, international investment, tourism and major events in the state of New South Wales, Australia.', ' The current Minister for Trade and Industry is Niall Blair, since 30 January 2017.', ' He is assisted by the Minister for Tourism and Major Events, currently Adam Marshall also since 30 January 2017.', ' Together the ministers administers these portfolios through the Department of Industry, Skills and Regional Development, known as the NSW Department of Industry, and also through Destination NSW, Venues NSW, and a range of small agencies.']], ['Willow Vale, New South Wales (Wingecarribee)', ['Willow Vale is a Northern Village of the Southern Highlands of New South Wales, Australia, in Wingecarribee Shire.', ' It is located 1 km north of Mittagong and is often considered part of Braemar along with its neighbour Balaclava.', ' At the 2016 census , Willow Vale had a population of 717.']], ['Willow Tree railway station', ['Willow Tree railway station is located on the Main Northern line in New South Wales, Australia.', ' It serves the village of Willow Tree, opening on 13 August 1877 as Warrah when the line was extended from Murrururundi to Quirindi.', ' It was renamed Willow Tree in 1879.']], ['Braemar, New South Wales', ['Braemar is a northern village of the Southern Highlands of New South Wales, Australia in Wingecarribee Shire.', ' It is located 2\\xa0km north-east of Mittagong and is often considered to include the hamlet villages of Balaclava and Willow Vale.']], ['Joseph Wild', ['Joseph Wild (also Wilde) (c.1759 or 1773–1847) was an early explorer of Australia.', ' He was sentenced on 21 August 1793 in Chester for burglary, together with his brother, George.', ' Both were transported to Australia as convicts in 1797, arriving in Port Jackson (Sydney) on the ship the \"Ganges\" on 2 June (George died in 1812).', ' He was under the charge of physician and pastoralist Charles Throsby and together they later became explorers in southern New South Wales.', ' In particular they were the first Europeans to explore the area that became the Australian Capital Territory and Wild was credited with the discovery of Lake George.', ' In 1810 he received a ticket of leave, and in January 1813 he was granted a conditional pardon.', ' On 9 December 1815 Wild was appointed first Constable of the Five Islands District (now Illawarra).', ' During the next few years he accompanied Throsby on many expeditions throughout New South Wales.', ' In 1819 he was granted 100 acre in Sutton Forest for services for Throsby and in 1821 he was appointed constable of the County of Argyle.', ' It is said that he and his wife Elizabeth had a large family.', ' He died on 25 May 1847 when he was gored by a bull at Wingecarribee Swamp.', ' He was the first person to be buried behind the church in the Bong Bong Cemetery, Moss Vale, New South Wales.']], ['Colo Parish', ['The Parish of Colo is a parish of the County of Camden in the Southern Highlands region of New South Wales.', ' It is centred on the town of Colo Vale, and includes Aylmerton, Willow Vale, Alpine and Yerrinbool.', ' It also includes the northern parts of Mittagong that are north of the Old Hume Highway.', ' The new Hume Highway runs through the parish from south-west to north-east.']], ['Unanderra–Moss Vale railway line', ['The Unanderra–Moss Vale railway line is a cross country railway line in New South Wales, Australia.', ' The line branches from the Illawarra line at Unanderra and winds west up the Illawarra escarpment to join the Main South line at Moss Vale.', ' The line is one of the most scenic in New South Wales, and for the first 20\\xa0km after leaving Unanderra has an almost continuous grade 1 in 30 providing spectacular view over the Illawarra coastline.']], ['Willow Vale, New South Wales (Kiama)', ['Willow Vale is a small town in New South Wales, Australia, in the Municipality of Kiama.', ' It is made up of residences, dairy farms, and more recently the Crooked River Winery.']], ['Mittagong', ['Mittagong is a town located in the Southern Highlands of New South Wales, Australia, in Wingecarribee Shire.', ' The town acts as the gateway to the Southern Highlands when coming from Sydney.', ' Mittagong is situated at an elevation of 635 m .', ' The town is close to Bowral, Berrima, Moss Vale and the Northern Villages such as Yerrinbool and Colo Vale.', ' Moreover, Mittagong is home to many wineries of the Southern Highlands which has been a recent growing wine and cellar door region.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.659\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abdcea955429965af743e5a', 'answer': 'White Knights', 'question': 'David Wayne Hull (born 1962 or 1963) is a leader of which group that originated in Mississippi in the early 1960s under the leadership of Samuel Bowers, its first Grand Wizard?', 'supporting_facts': [['David Wayne Hull', 0], ['White Knights of the Ku Klux Klan', 0]], 'context': [['Westside High School (Jacksonville)', ['Westside High School is a public high school in Jacksonville, Florida.', \" It is part of the Duval County School District and serves Jacksonville's Westside.\", ' The school was established in 1959 and was originally named Nathan B. Forrest High School, after Nathan Bedford Forrest, a Confederate general and first Grand Wizard of the Ku Klux Klan.', ' The fact that the school was named for Forrest was a point of significant controversy until the Duval County School Board changed the name in 2014.']], ['Samuel Green (Ku Klux Klan)', ['Samuel Green (13 November 1889 – 18 August 1949) was an Grand Wizard of the Knights of the Ku Klux Klan in the late 1940s, organizing its brief reformation.']], ['European-American Unity and Rights Organization', ['The European-American Unity and Rights Organization (EURO) is an American organization led by former Grand Wizard of the Knights of the Ku Klux Klan, David Duke.', ' Founded in 2000, the group has been described as white nationalist and white supremacist.']], ['White Knights of the Ku Klux Klan', ['The White Knights of the Ku Klux Klan are considered the most militant as well as the most violent chapter of the Ku Klux Klan in history.', ' They originated in Mississippi in the early 1960s under the leadership of Samuel Bowers, its first Grand Wizard.', ' The White Knights of Mississippi were formed in 1964, and they included roughly 200 members of the Original Knights of Louisiana.', ' The White Knights were not interested in holding public demonstrations or in letting any information about themselves get out to the masses.', ' Similar to the United Klans of America (UKA), the White Knights of Mississippi were very secretive about their group.', ' Within a year, their membership was up to around six thousand, and they had Klaverns in over half of the counties in Mississippi.', ' But by 1967, the number of active members had shrunk to around four hundred.']], ['Omeria Scott', ['Omeria McDonald Scott (born November 21, 1956) is an American Democratic politician.', ' She is a member of the Mississippi House of Representatives from the 80th District, being first elected in 1992.', ' She was also an award winning cheerleader for R.H. Watkins High School in Laurel, MS in the early days of integration.', ' In high school she was a positive force in bridging relationships in the greater community of Laurel, MS at a time when it was especially dangerous to do so, given that the grand wizard of the Ku Klux Klan, Devours Nix, lived in Laurel, MS at this time.']], ['Leaders of the Ku Klux Klan', ['The national leader of the Ku Klux Klan is called either a Grand Wizard or an Imperial Wizard, depending on which KKK organization is being described.']], ['David Wayne Hull', ['David Wayne Hull (born 1962 or 1963) is a leader of the White Knights of the Ku Klux Klan, which is considered the most militant as well as the most violent Ku Klux Klan in history.']], ['Tom Metzger', ['Thomas Linton Metzger (born April 9, 1938) is an American white supremacist, skinhead leader and former Klansman.', ' He founded White Aryan Resistance (WAR).', ' He was a Grand Wizard of the Ku Klux Klan in the 1970s.', ' Metzger has voiced strong opposition to immigration to the United States.', ' In the early 1980s, he was registered with the Democratic Party and sought to be a Democratic candidate for the United States House of Representatives and Senate.', ' He has been incarcerated in Los Angeles County, California, and in Toronto, Canada, and has been the subject of several lawsuits and government inquiries.', ' He, his son John, and WAR were fined $12 million as a result of the murder of an Ethiopian by skinheads affiliated with WAR.']], ['Samuel Bowers', ['Samuel Holloway Bowers (August 25, 1924 – November 5, 2006), Former Ku Klux Klan Imperial Wizard, was a convicted murderer and leading white supremacist activist in Mississippi during the Civil Rights Movement.', ' In response to this movement, he co-founded a reactionary organization, the White Knights of the Ku Klux Klan.', ' Bowers committed two notorious murders of civil rights activists in southern Mississippi: The 1964 murders of Chaney, Goodman, and Schwerner near Philadelphia, for which he served six years in federal prison; and the 1966 murder of Vernon Dahmer in Hattiesburg, for which he was sentenced to life in prison 32 years after the crime.', ' He also was accused of bombings of Jewish targets in the cities of Jackson and Meridian in 1967 and 1968 (according to the man who was convicted of some of the bombings, Thomas A. Tarrants III).', ' He died in prison at the age of 82.']], ['Forrest School (Chapel Hill, Tennessee)', ['Forrest School is a public school in Chapel Hill, Tennessee.', ' It serves grades 7-12 and is part of the Marshall County School District.', ' The school is also known as Forrest Middle School for grades 7-8 and Forrest High School for grades 9-12.', ' It is named for Nathan Bedford Forrest, a Confederate general and first Grand Wizard of the Ku Klux Klan, who was born in Chapel Hill.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.660\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab5f249554299488d4d9a5d', 'answer': 'Yamaha 600 class', 'question': 'What class of yamaha class sport bike motorcycle did Ryuji Yokoa use when he finished as champion in the MFJ All Japan Road Race GP250 Championship in 2006?', 'supporting_facts': [['Ryuji Yokoe', 3], ['Yamaha YZF-R6', 0]], 'context': [['Yuuki Ito', ['Yuki Ito (伊藤 勇樹 , Itō Yūki ) is a Grand Prix motorcycle racer from Japan.', ' He currently races in the Asia Road Racing SS600 Championship and the All Japan Road Race J-GP2 Championship aboard a Yamaha YZF-R6.', ' He has also competed in the MFJ All Japan JSB1000 Championship, the MFJ All Japan Road Race GP250 Championship, and the East Japan GP250 Challenge Cup, which he was champion of in 2007.']], ['Ryuji Yokoe', ['Ryuji Yokoe (横江竜司 , Yokoe Ryūji , born 18 January 1978) is a Japanese motorcycle racer.', ' He currently competes in the All Japan Road Race ST600 Championship aboard a Yamaha YZF-R6.', ' Fastest lap record holder in Sugo race way on ST600, he won the MFJ All Japan Road Race ST600 Championship in 2015 for the Yamaha Thailand Racing Team aboard a Yamaha YZF-R6.', ' Yokoe has previously competed in the MFJ All Japan Road Race GP125 Championship, the MFJ All Japan Road Race GP250 Championship – where he finished as champion in 2006 – and the MFJ All Japan Road Race JSB1000 Championship.']], ['Takumi Takahashi', ['Takumi Takahashi (高橋 巧 , Takahashi Takumi , born 26 November 1989) is a Japanese motorcycle racer.', ' He races in the MFJ All Japan Road Race JSB1000 Championship for the MuSASHi RT HARC-PRO.', ' Honda team.', ' Takahashi has also competed in the MFJ All Japan Road Race GP125 Championship and the MFJ All Japan Road Race GP250 Championship, where he was champion in 2008.', ' He won the Suzuka 8 Hours in 2010 with Ryuichi Kiyonari and Takaaki Nakagami and in 2013 and 2014 with Michael van der Mark and Leon Haslam.', ' A test rider for Honda Racing Corporation, Takahashi in 2015 made a wild card appearance in his home race in the MotoGP World Championship.']], ['Taro Sekiguchi', ['Taro Sekiguchi (関口 太郎 , Sekiguchi Tarō , born December 5, 1975 in Fuchu, Tokyo, Japan) is a Japanese motorcycle road racer.', ' He was the MFJ All Japan Road Race GP250 champion in 2001 and the European 250cc champion in 2003.']], ['Kenta Fujii', ['Kenta Fujii (born 4 April 1994 in Suzuka) is a Japanese Grand Prix motorcycle racer.', ' He currently competes in the All Japan Road Race JP250 Championship aboard a Honda CBR600RR.', ' He has previously competed in the MFJ All Japan Road Race GPMono Championship, the MFJ All Japan Road Race GP125 Championship, the MFJ All Japan Road Race J-GP3 Championship and the Spanish CEV Moto3 Championship.', ' Fujii won the GPMono title in 2010, and the J-GP3 title in 2011.']], ['Yuma Yahagi', ['Yuma Yahagi (矢作 雄馬 , Yahagi Yūma , born 26 July 1990) is a Japanese motorcycle racer.', ' He has competed in the MFJ All Japan Road Race GP125 Championship, the MFJ All Japan Road Race J-GP3 Championship and the MFJ All Japan Road Race ST600 Championship.']], ['Sasuke Shinozaki', ['Sasuke Shinozaki (篠崎 佐助 , Shinozaki Sasuke , born 8 June 1993) is a Japanese motorcycle racer.', ' He has competed in the MFJ All Japan Road Race GP125 Championship, the Red Bull MotoGP Rookies Cup, the MFJ All Japan J-GP3 Championship and the MFJ All Japan Road Race ST600 Championship.']], ['Kazuki Watanabe (motorcycle racer)', ['Kazuki Watanabe (渡辺 一樹 , Watanabe Kazuki , born 2 October 1990) is a Japanese motorcycle racer.', ' In 2017 he competes in the Supersport World Championship aboard a Kawasaki ZX-6R.', ' He has also competed in the MFJ All Japan Road Race JSB1000 Championship, the MFJ All Japan Road Race GP250 Championship and the MFJ All Japan Road Race J-GP2 Championship, where he was champion in 2012.']], ['Yamaha YZF-R6', ['The Yamaha YZF-R6 is a Yamaha 600 class sport bike motorcycle, first introduced in 1998, updated in 2001, 2003, 2006, 2008, 2017, and revised in the years in between.']], ['Tatsuya Yamaguchi (motorcycle racer)', ['Tatsuya Yamaguchi (山口 辰也 , Yamaguchi Tatsuya , born 11 February 1976) is a Japanese motorcycle racer.', ' He currently races in the MFJ All Japan Road Race Championship JSB1000 class aboard a Honda CBR1000RR and the Asia Road Race SS600 Championship aboard a Honda CBR600RR.', ' He has also competed in the MFJ All Japan Road Race GP250 Championship, the MFJ All Japan Road Race JSB1000 Championship (where he was champion in 2002), the MFJ All Japan Road Race J-GP2 Championship, and the MFJ All Japan Road Race ST600 Championship, where he was champion in 2010 and 2011.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.661\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7914e155429974737f7973', 'answer': 'March 30, 1983', 'question': 'What is the birthday of the oldest among Vicente García and Ximena Sariñana?', 'supporting_facts': [['Vicente García (musician)', 0], ['Ximena Sariñana', 0]], 'context': [['Ximena Sariñana (album)', ['Ximena Sariñana is the eponymous second studio album released by Mexican singer Ximena Sariñana.', ' After the success of \"Mediocre\", with sales of more than 100,000 copies in Mexico, Warner Music Group producers proposed that Ximena Sariñana record a second album.', ' The album contains ten songs in English and one in Spanish, the latter produced by Natalia Lafourcade.', ' Sariñana entered the studio in 2009 when she began recording a song for the soundtrack of the second installment of the \"Twilight\" series, .']], ['Mediocre (album)', ['Mediocre (] ) is the Grammy Award and Latin Grammy nominated debut album of Mexican singer and actress Ximena Sariñana, released in the United States on \\xa015,\\xa02008\\xa0(2008--) .', ' \"Mediocre\" reached #10 on Billboard\\'s Latin Pop Albums, and spawned the hit single \"Vidas Paralelas\" (\"Parallel Lives\").', ' The album gained additional attention when the iTunes Store featured Sariñana\\'s song \"Normal\" as the \"canción de la semana\" — iTunes\\' free Latino song of the week — during the week of \\xa008,\\xa02008\\xa0(2008--) .']], ['Los Sueños de un Hígado', ['Los Sueños de un Hígado (English: The Dreams of a Liver ) is a live album by Omar Rodriguez Lopez Group which was recorded for the BBC on March 11, 2009 at Maida Vale Studios, England.', ' The album features Omar Rodríguez-López, Thomas Pridgen, Juan Alderete de la Peña, Marcel Rodriguez-Lopez, Ximena Sariñana and Mark Aanderud and was released digitally via Rodriguez-Lopez Productions on September 27, 2009.', ' A limited edition of 1,000 vinyl was released on December 1.']], ['Vicente García González', [\"Vicente García González was a General in the Cuban Ten Years' War (Spanish: Guerra de los Diez Años , also known as the Great War) and later a Cuban President who was assassinated by the Spanish after the war.\", ' García was born on January 23, 1833 in Las Tunas, and died on May 4, 1886.']], ['Premios Oye! 2008', ['The 7th Annual Premios Oye!', ' took place at the Foro Monumental in Zacatecas, Zacatecas on November 26, 2008.', ' The nominees were announced on September 29 with Vicente Fernández receiving 5 nominations, followed by Julieta Venegas and Juanes with 4 each one, with 3 each one, Amandititita and Ximena Sariñana.', ' Miguel Bosé will be awarded by the Academia Nacional de la Música en México for his 31 years or career.', ' The voting process is certified by PricewaterhouseCoopers.']], ['Dōitashimashite', ['Dōitashimashite (どういたしまして , \"You\\'re Welcome\") is a live album released by artist Omar Rodríguez-López which was recorded in USA over four nights in September 2010.', ' It is the second live album to be released by the Omar Rodriguez Lopez Group, with the first being the live BBC studio session \"Los Sueños de un Hígado\".', ' Notably, during the week of these four concerts vocalist Ximena Sariñana was said to be suffering from laryngitis, a cold, and the flu.']], ['New Mexico Wool Manufacturing Company', ['The article of incorporation for the New Mexico Wool Manufacturing Company was introduced and passed in the New Mexico Territorial Legislature on January 30, 1861.', ' Its founding associates were Ceran St. Vrain, José Guadalupe Gallegos, Oliver P. Hovey, Anastacio Sandoval, Rafael Armijo, José Manuel Gallegos, Hamilton G. Fant, Nazario Gonzales, J. Francisco Chaves; Levi Spiegelberg, A.P. Wilbar, Miguel A. Otero, William W. Griffin; José Leandro Perea (brother of Juan Perea, who was the father of Colonel Francisco Perea); S. J. Spiegelberg, Tomás Cabeza de Baca, Sidney A. Hubbell, Francisco Lopez, William A. Street, Ramon Luna; Miguel E. Pino, who became commander of 2nd New Mexico Volunteer Infantry during the Civil War; Thomas H. Hopkins; Simon Delgado (cousin of Miguel E. Pino) who, with his mother, Doña Maria de la Luy Baca de Delgado, purchased the \"Yglesia Castrense\" in Santa Fe from Bishop Lamy in exchange for $1,000 and a portion of the site for St. Michael\\'s College (San Miguel College) in Santa Fe; M. Steck, Vicente García, Teodoro Baca, Vicente Romero, José Jaramillo, and Manuel Vigil.', ' They claimed lawful use, occupation, and right to construct roads and erect buildings on, any wild lands within the Territory not the property of other individuals or corporations.', ' They also claimed lawful use of (but not diversion of or injure others use of) water and the right to construct machinery on any river or stream upon said lands.', ' The statement of capital stock was 3,500 shares at $100 each, or $350,000, with the right to increase the number of total shares to 7,500 at a value of $750,000.']], ['Juan Campodónico', ['Juan Campodónico (Montevideo, Uruguay, 1971), sometimes working under his stage name Campo, is an Uruguayan musician, producer, composer, creator and former member of El Peyote Asesino, Bajofondo and Campo.', ' He produced albums by Jorge Drexler (Frontera, Sea, Eco, 12 Segundos de Oscuridad), Luciano Supervielle, Bajofondo (Tango Club, Mar Dulce, Presente), El Cuarteto de Nos (Raro, Bipolar, Porfiado), La Vela Puerca (El impulso), OMAR, Sordromo, No Te Va Gustar, Santullo and Ximena Sariñana among others.', ' He created the Bajofondo project alongside iconic producer and two-time Academy Award winner for Best Original Score Gustavo Santaolalla (Brokeback Mountain and Babel).', ' He has been awarded with various Latin Grammy, Premios Gardel and Graffiti awards for his work as a producer, as well as with many golden records.']], ['Ximena Sariñana', ['Ximena Sariñana Rivera (] ; born October 29, 1985) is a Mexican singer-songwriter and actress.', ' In 2009, she received critical acclaim and a Grammy nomination for her debut album, \"Mediocre\".']], ['Vicente García (musician)', ['Vicente García (born March 30, 1983) is a Dominican singer, songwriter and composer.', ' He is the former lead singer of the Dominican alternative rock band Calor Urbano, which he left in 2010 to pursue a solo career.', ' Garcia has collaborated in concerts with renowned artists like Juan Luis Guerra, Alejandro Sanz, Cultura Profetica, Juanes, Ximena Sariñana and Maná among others.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.661\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a710bb15542994082a3e50d', 'answer': 'Dan Bilzerian', 'question': 'Which American professional poker player also starred in the 2015 movie \"Extraction\"?', 'supporting_facts': [['Extraction (film)', 0], ['Extraction (film)', 1], ['Dan Bilzerian', 0]], 'context': [['Extraction (film)', ['Extraction is a 2015 American action-thriller film directed by Steven C. Miller and written by Umair Aleem.', ' The film stars Kellan Lutz, Bruce Willis, Gina Carano, D. B. Sweeney, Dan Bilzerian and Steve Coulter.', ' The film was released on December 18, 2015, in a limited release, and through video on demand by Lionsgate Premiere.']], ['Howard Lederer', ['Howard Henry Lederer (born October 30, 1963) is an American professional poker player.', ' He has won two World Series of Poker bracelets and holds two World Poker Tour titles.', ' Lederer has also contributed to several books on poker strategy and has provided commentary for poker programming.', ' He is known by poker fans and players as \"The Professor\" and is the older brother of professional poker player Annie Duke.']], ['Dag Palovic', ['Dag Palovič (* 4 January 1975, Bratislava) is a Slovak professional poker player, businessman and a former TV host.', ' Since 1 January 2011, he is a member of PokerStars Team Pro, first and as of October 2011 only sponsored poker player from Slovakia.', ' He is best known for making two European Poker Tour (EPT) final tables as well as being the only player from Slovakia who has cashed in the World Series of Poker (WSOP) Main Event, finishing 120th in 2009 and 37th in 2010.', ' As of March 2013, he is second leading Slovak all time money list with career earnings of $909,405 and is also an author of first Slovak poker book on poker titled \"Ako sa stať poker pro\" (How to become a poker pro), co-authored by 1983 World Series Of Poker champion Tom McEvoy.', ' From 2000 until 2004 he was CEO and Chairman Of The Board of Directors of \"ad pepper media Slovakia, a.s.\", the daughter company of one of the world´s leading e-Adverising german-dutch agency \"ad pepper media International N.V.\" for Slovak and Czech Republic.']], ['Tommy Angelo (poker player)', ['Tommy Angelo (born August 25, 1958) is an Oakland, California professional poker player, writer, and coach.', \" Angelo was a career musician in the 1980's, performing rock and country on drums and piano.\", ' In 1990, he became a full-time professional poker player.', ' Since then has since written 100 magazine articles, written and produced 18 poker training videos, and written and published three books on poker.']], ['Ben Lamb', ['Benjamin \"Ben\" Lamb (born March 31, 1985) is an American professional poker player.', ' Lamb was the 2011 World Series of Poker Player of the Year.', \" He was also a member of the 2011 November Nine, finishing in third place in the no limit hold'em championship event.\", \" Lamb has one World Series of Poker bracelet and five career World Series of Poker (WSOP) final tables, three in variations of Pot Limit Omaha, one in no limit hold'em and one in the 8-game mix format.\", ' He was the winner of the 2011 \"Card Player\" Player of the Year Award.']], ['Billy Baxter (poker player)', ['William E. Baxter, Jr. (born 1940) is an American professional poker player and sports bettor.', ' He has won numerous tournament titles in his career as a professional poker player, including seven World Series of Poker bracelets.']], ['Viacheslav Zhukov', ['Viacheslav Zhukov (c. 1989) is a Russian professional poker player who has won two World Series of Poker bracelets. Prior to becoming a professional poker player, Zhukov was a geologist in Russia.', ' he has career earnings of $940,000, $838,000 of which was earned at the World Series of Poker.']], ['Dan Bilzerian', ['Dan Brandon Bilzerian (born December 7, 1980) is an American professional poker player.']], ['Vanessa Rousso', ['Vanessa Ashley Rousso (born February 5, 1983) is an American professional poker player.', ' Born in White Plains, New York, Rousso holds dual citizenship with the United States and France.', ' Rousso was a member of Team PokerStars from 2006 to 2015, with the online name Lady Maverick.', ' She is a spokesperson for GoDaddy.com.', \" She has earned money as a professional poker player since 2005, and has become one of the game's sex symbols.\"]], ['Andy Frankenberger', ['Andy Frankenberger is a professional poker player and former equity derivatives trader from New York City.', ' In his first year as a professional poker player, Frankenberger was named World Poker Tour (WPT) Season IX Player of the Year.', \" Card Player Magazine described this as one of poker's best rookie years in a September 2011 cover story.\", ' Frankenberger followed this up by winning back to back bracelets at the World Series of Poker in 2011 and 2012.', ' He has been prominently featured in financial media including The Wall Street Journal, Fox Business Network, and Bloomberg Television.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.662\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a762d53554299109176e6a7', 'answer': 'Croatian', 'question': \"What was the the origin of the player that the Bull's traded with 76ers during the 1999-2000 midseason?\", 'supporting_facts': [['1999–2000 Chicago Bulls season', 4], ['Toni Kukoč', 0]], 'context': [['List of Philadelphia 76ers head coaches', ['The Philadelphia 76ers are an American professional basketball team based in Philadelphia, Pennsylvania.', ' They are a member of the Atlantic Division of the Eastern Conference in the National Basketball Association (NBA).', ' Formerly known as the Syracuse Nationals, the 76ers joined the NBA when it was founded in 1949.', ' The Nationals had a record of 51–13 in their first NBA season under coach Al Cervi and won the Eastern Division crown.', ' The franchise were purchased by Philadelphian Irv Kosloff and Ike Richma in the spring of 1963; the NBA approved their franchise shift on May 22 and name change to the Philadelphia 76ers on August 6.', ' This brought professional basketball back to the city, which had been without a team since the Golden State Warriors left Philadelphia in 1962.', ' After coaching the 76ers since , Doug Collins resigned as head coach on April 18, 2013 following the 2012–13 season.', ' Brett Brown was hired to be the head coach of the 76ers on August 15, 2013 prior to the start of the 2013-14 season.']], ['1997–98 Detroit Pistons season', [\"The 1997–98 NBA season was the Pistons' 50th season in the National Basketball Association, and 41st season in the city of Detroit.\", ' Despite signing free agents Brian Williams and Malik Sealy during the offseason, the Pistons got off to a slow start with a 6–11 record as Joe Dumars missed ten games due to hamstring and shoulder injuries.', ' In late December, they traded Theo Ratliff and Aaron McKie to the Philadelphia 76ers for Jerry Stackhouse and Eric Montross.', ' At midseason, head coach Doug Collins was fired after a 21–24 start, and was replaced with Alvin Gentry.', ' Collins would later on get a job as color analyst for the \"NBA on NBC\".', ' Despite another stellar season from Grant Hill, who was selected for the 1998 NBA All-Star Game, the Pistons missed the playoffs finishing sixth in the Central Division with a 37–45 record.', ' Following the season, Sealy signed as a free agent with the Minnesota Timberwolves, Grant Long re-signed with the Atlanta Hawks, and Rick Mahorn re-signed with the Philadelphia 76ers.']], ['1983–84 Philadelphia 76ers season', [\"The 1983–84 NBA season was the 76ers' 35th season in the NBA and 21st season in Philadelphia.\", ' The 76ers entered the season as the defending NBA Champions, having won the NBA Championship the year prior, sweeping the Los Angeles Lakers in four games.', ' The team would start fast posting 21 wins in their first 26 games but finished with a 52-30 record.', ' The major difference was that they were just around .500 on the road for the year, unlike the previous season, where they won 30 regular season games away from Philadelphia.', ' The 76ers would lose in the first round of the newly expanded playoff format to the New Jersey Nets, who had never won a playoff series in their NBA history to that point.', ' The 76ers lost all three post season games at The Spectrum.']], ['1999–2000 Chicago Bulls season', [\"The 1999–2000 NBA season was the Bulls' 34th season in the National Basketball Association.\", ' The Bulls won the Draft Lottery, and selected Elton Brand out of Duke University with the first pick in the 1999 NBA draft.', ' During the offseason, the team acquired Hersey Hawkins from the Seattle SuperSonics, and re-signed free agents B.J. Armstrong and Will Perdue, who both won championships with the team in the early 1990s.', \" However, the Bulls' struggles continued as they lost 26 of their first 28 games.\", ' At midseason, Toni Kukoč was traded to the Philadelphia 76ers, as the team acquired John Starks from the Golden State Warriors in a three-team trade.', ' However, after just four games, Starks was released.']], ['Joel Embiid', ['Joel Hans Embiid ( ; born 16 March 1994) is a Cameroonian professional basketball player for the Philadelphia 76ers of the National Basketball Association (NBA).', ' After one year of college basketball at the University of Kansas, he was drafted with the third overall pick in the 2014 NBA draft by the 76ers.', ' He has nicknamed himself \"The Process\" in response to a refrain from 76ers fans during the Sam Hinkie-era to \"trust the process\".']], ['1999–2000 Philadelphia 76ers season', [\"The 1999–2000 NBA season was the 76ers' 51st season in the National Basketball Association, and 37th season in Philadelphia.\", ' During the offseason, the Sixers signed free agent Bruce Bowen while acquiring Billy Owens from the Orlando Magic, who acquired him from the Seattle SuperSonics.', ' At midseason, the Sixers traded Owens along with second-year guard Larry Hughes to the Golden State Warriors, and traded Bowen to the Chicago Bulls for Toni Kukoč in a three-team trade.', ' However, Bowen was released and signed with the Miami Heat.', ' The Sixers won seven of their final nine games, and finished third in the Atlantic Division with a 49–33 record.', ' Allen Iverson averaged 28.4 points per game, and made his first All-Star appearance as he was selected for the 2000 NBA All-Star Game in Oakland.', ' In the first round of the playoffs, the Sixers defeated the 4th-seeded Charlotte Hornets in four games, but would lose in six games to the Indiana Pacers in the semifinals.']], ['2000–01 Atlanta Hawks season', [\"The 2000–01 NBA season was the Hawks' 52nd season in the National Basketball Association, and 33rd season in Atlanta.\", ' Under new head coach Lon Kruger, the Hawks got off to a bad start losing their first seven games, but then posted a 7–7 record in December.', ' In January, they traded Jim Jackson to the Cleveland Cavaliers for Brevin Knight.', ' After playing in the 2001 NBA All-Star Game, Dikembe Mutombo was traded at midseason along with Roshown McLeod to the Philadelphia 76ers for Theo Ratliff, Toni Kukoč and Nazr Mohammed.', ' However, Ratliff injured his shooting hand prior to the trade, and was out for the remainder of the season.', ' Mutombo would later on be named Defensive Player of the Year with the Sixers, who went on to lose in five games to the Los Angeles Lakers in the NBA Finals.']], ['1976 NBA draft', ['The 1976 NBA draft was the 30th annual draft of the National Basketball Association (NBA).', ' The draft was held on June 8, 1976, before the 1976–77 season.', ' In this draft, 18 NBA teams took turns selecting amateur U.S. college basketball players and other eligible players, including international players.', ' The first two picks in the draft belonged to the teams that finished last in each conference, with the order determined by a coin flip.', ' The Atlanta Hawks won the coin flip and were awarded the first overall pick, while the Chicago Bulls were awarded the second pick.', ' The Hawks then traded the first pick to the Houston Rockets before the draft.', ' The remaining first-round picks and the subsequent rounds were assigned to teams in reverse order of their win–loss record in the previous season.', ' The New York Knicks forfeited their first-round draft pick due to their illegal signing of George McGinnis whose rights were held by the Philadelphia 76ers.', ' The 76ers, the Golden State Warriors and the Buffalo Braves also forfeited their second, third and fourth-round pick respectively due to their participation in 1975 supplementary draft American Basketball Association (ABA) players who had never been drafted in the NBA.', ' A player who had finished his four-year college eligibility was eligible for selection.', ' If a player left college early, he would not be eligible for selection until his college class graduated.', ' Before the draft, 26 college underclassmen were declared eligible for selection under the \"hardship\" rule.', ' 13 of them withdrew before the draft, leaving only 13 early entry candidates eligible for selection.', ' These players had applied and gave evidence of financial hardship to the league, which granted them the right to start earning their living by starting their professional careers earlier.', ' The draft consisted of 10 rounds comprising the selection of 173 players.', ' On August 8, 1976, the league also hosted a Dispersal draft for ABA players from the Kentucky Colonels and Spirits of St. Louis, who were not included in the ABA–NBA merger.']], ['1998–99 Philadelphia 76ers season', ['The 1998–99 NBA season was the 76ers 50th season in the National Basketball Association, and 36th season in Philadelphia.', ' After a four-month lockout wiped out half the season, the Sixers signed free agents Matt Geiger and George Lynch, while re-signing former 76ers forward Rick Mahorn.', ' At midseason, they traded second-year forward Tim Thomas and Scott Williams to the Milwaukee Bucks for Tyrone Hill.', ' The Sixers recorded their first winning month in five years winning 8 of 13 games in February, on their way to making the playoffs for the first time in eight years with a 28–22 record, third in the Atlantic Division.', ' Allen Iverson led the league in scoring averaging 26.8 points per game.', ' In the first round of the playoffs, the Sixers defeated the 3rd–seeded Orlando Magic in four games, but were swept in the semifinals by the Indiana Pacers in four straight games.', ' Following the season, Mahorn retired after making his second stint with the Sixers.']], ['Toni Kukoč', ['Toni Kukoč (] ; born September 18, 1968) is a Croatian retired professional basketball player who is currently Special Advisor to Jerry Reinsdorf, the owner of the Chicago Bulls.', ' After a highly successful period in European basketball, he was one of the first established European stars to play in the National Basketball Association (NBA).', ' He won the NBA Sixth Man of the Year Award in 1996.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.662\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8b67b655429950cd6afcde', 'answer': 'Jürgen M. Geissinger', 'question': \"Who is the controlling shareholder of the world's fourth-largest tyre manufacturer?\", 'supporting_facts': [['Juergen M. Geissinger', 0], ['Juergen M. Geissinger', 4], ['Continental AG', 2]], 'context': [['Stomil Olsztyn (company)', ['Stomil Olsztyn was a tyre manufacturer based in Olsztyn, Poland.', ' The company came into existence when the tyre plant OZOS „Stomil”, founded in 1967, was privatized in 1992.', ' In 1995 Michelin acquired the majority share in Stomil.', ' From 1995 to May 28, 2004 Stomil was quoted at the Warsaw Stock Exchange.', ' In 2005 Michelin gained full control of Stomil and renamed it to \"Michelin Polska S.A.\".', ' With roughly 4,000 employees \"Michelin Polska\" is one of the largest Michelin plants and the largest tyre plant in Poland.']], ['1978 Brazilian Grand Prix', ['The 1978 Brazilian Grand Prix was a Formula One motor race held on 29 January 1978 at Jacarepagua.', ' The race was won by Argentine driver Carlos Reutemann driving a Ferrari 312T2 in a flag-to-flag performance.', ' The win also represented the first win for tyre manufacturer Michelin.', ' Local driver Emerson Fittipaldi was second, scoring the first podium finish for the Fittipaldi Automotive with Austrian Brabham driver Niki Lauda finishing third.', ' French driver Didier Pironi took his first points in Formula One, finishing sixth.', ' The race also represented the first win for French tyre manufacturer Michelin in Formula One.']], ['JK Tyre', ['JK Tyre & Industries Ltd is an Automotive Tyre, Tubes and flaps manufacturing company based in Delhi, India.', ' The name JK is derived from the initials of Kamlapatji (1884–1937) and his father Seth Juggilal (1857–1922).', ' The company is the market leader in Truck/Bus Radial tire in India and is the only tyre manufacturer offering the entire range of 4 wheeler radials for Trucks, Buses and Cars.', ' JK Tyre has a worldwide customer base in over 80 countries across all 6 continents.', ' It is a part of J. K. Organisation group of Companies.', ' JK Tyre acquired Mexican tyre major – Tornel in 2008.', ' With production facilities in all 9 plants, total production capacity is almost 20 million tyres p.a.']], ['Clément Tyres', ['Clément Tyres, Clément Pneumatics, Clément Pneumatici, is a Franco Italian tyre manufacturer that was founded by French industrialist and bicycle manufacturer Adolphe Clément-Bayard, possibly around the 1900s.', ' The brand ceased active trading in the 1990s but was revived under American identity in 2010.']], ['Continental AG', ['Continental AG, commonly known as Continental, is a leading German automotive manufacturing company specialising in tyres, brake systems, interior electronics, automotive safety, powertrain and chassis components, tachographs, and other parts for the automotive and transportation industries.', ' Continental is based in Hanover, Lower Saxony, Germany.', \" Continental is the world's fourth-largest tyre manufacturer.\", ' Continental was founded in 1871 as a rubber manufacturer, \"Continental-Caoutchouc und Gutta-Percha Compagnie\".', \" After acquiring Siemens AG's VDO automotive unit in 2007 Continental was ranked third in global OEM automotive parts sales in 2012 according to a study sponsored by PricewaterhouseCoopers.\"]], ['TerreStar Corporation', ['TerreStar Corporation (\"TSTR\"), formerly \"Motient Corp.\" (MNCP - 2000-2007) and \"American Mobile Satellite Corp.\" (AMSC - 1988-2000), was the controlling shareholder of \"TerreStar Networks Inc.\", \"TerreStar National Services, Inc.\" and \"TerreStar Global Ltd.\", and a shareholder of SkyTerra Communications.']], ['Lapo Elkann', ['Lapo Edovard Elkann (born 7 October 1977) is an Italian entrepreneur and grandson of Gianni Agnelli, the former controlling CEO and controlling shareholder of Fiat Automobiles.']], ['Belshina', ['Belshina is a tyre manufacturer in Belarus.', ' The name is an abbreviation for \"Belаruskaya shina\", or \"Belarusian Tyre\".']], ['Juergen M. Geissinger', ['Jürgen M. Geissinger (born July 24, 1959) is a German technology business executive and Chief Executive Officer of \"Senvion S.A.\", a Hamburg based wind turbine manufacturer.', ' Geissinger is best known for his role as the Chief Executive of Schaeffler Technologies AG & Co.', ' KG, a technology conglomerate known for its bearing solutions and precision components for engine and transmission systems for automotive, as well as industrial and aerospace applications.', ' During Geissinger’s tenure as CEO, annual sales have risen more than fivefold.', ' Schaeffler AG, employing over 76,000 people across 180 locations in 50 countries, with annual sales of $14 billion, is also the controlling shareholder of Continental AG with 49.9% of its shares.']], ['Neeraj Kanwar', [\"Neeraj Kanwar (born 6 September 1971) is the vice chairman and managing director of Apollo Tyres, India's second largest tyre manufacturer with annual revenues of over $2 billion.\", ' He is credited with turning Apollo Tyres from a commercial vehicle-focussed tyre manufacturer in India when he joined in 1995 into a multinational company that now manufactures tyres for commercial and passenger vehicles as well as two wheeler tyres.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.663\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abbc4d255429931dba144fe', 'answer': 'Weldenia', 'question': 'Which genus of plant grows originally in Mexico and Guatemala, Phylica or Weldenia?', 'supporting_facts': [['Phylica', 0], ['Phylica', 2], ['Weldenia', 1]], 'context': [['Mendoncia velloziana', ['Mendoncia velloziana is a plant native to Atlantic Coast restingas vegetation which is an ecosystem of Atlantic Forest biome.', ' In addition, this plant grows either in Cerrado vegetation of Brazil.', ' This plant grows in following states of Brazil: Bahia, Ceará Minas Gerais Rio de Janeiro, São Paulo, Paraná and Santa Catarina, and it is usually visited by the hummingbirds.']], ['Agave ghiesbreghtii', ['Agave ghiesbreghtii is an evergreen plant belonging to the family Asparagaceae, subfamily Agavoideae.', ' The plant grows in clustering rosettes, up to 75\\xa0cm in diameter and 50\\xa0cm tall with wide leaves which are guttered on top.', ' In spring the plant produces dense greenish brown to purple flowers on the top half of the unbranched spike which measures between 2.5m - 5m tall.', ' The species is endemic in Guatemala and the State of Mexico in Mexico.']], ['Weldenia', ['Weldenia is a monotypic genus of flowering plant in the Commelinaceae family, first described in 1829.', ' It has one single species: Weldenia candida, which grows originally in Mexico and Guatemala.']], ['Pinguicula orchidioides', ['Pinguicula orchidioides is a perennial rosette-forming insectivorous herb native to Mexico and Guatemala.', ' A species of butterwort, it forms summer rosettes of flat, succulent leaves up to 5\\xa0centimeters (4\\xa0in) long, which are covered in mucilagenous (sticky) glands that attract, trap, and digest arthropod prey.', ' Nutrients derived from the prey are used to supplement the nutrient-poor substrate that the plant grows in.', ' Uniquely among \"Pinguicula\" species from the Americas, \"p. orchidioides\" produces gemma-like basal buds which elongate into stolons and serve as a means of asexual reproduction.', ' In the winter the plant forms a non-carnivorous rosette of small, fleshy leaves that conserves energy while food and moisture supplies are low.', ' Single purple flowers appear between July and September on upright stalks up to 22 centimeters long.']], ['Salvia divinorum', [\"Salvia divinorum (also known as sage of the diviners, ska maría pastora, seer's sage, yerba de la pastora and just salvia) is a psychoactive plant which can induce visions and other spiritual experiences. Its native habitat is in cloud forest in the isolated Sierra Mazateca of Oaxaca, Mexico, where it grows in shady and moist locations.\", ' The plant grows to over a meter high, has hollow square stems, large leaves, and occasional white flowers with violet calyxes.', ' Botanists have not determined whether \"Salvia divinorum\" is a cultigen or a hybrid; native plants reproduce vegetatively, rarely producing viable seed.']], ['Argemone albiflora', ['Argemone albiflora, the white prickly poppy, also known as the bluestem prickly poppy or the Texas prickly poppy, is a small erect plant with a decorative white flower with a yellow latex.', ' It is deeply rooted with yellow or red stamens.', ' The plant is known for the sharp prickles on its stem and leaves.', ' The sepals fall off as the flower of this plant grows bigger.', ' It grows in the arid regions of the southern Midwest along roadsides and disturbed pieces of land.', ' Native Americans have long revered this plant for its medicinal and other uses.']], ['Pinguicula moranensis', ['Pinguicula moranensis is a perennial rosette-forming insectivorous herb native to Mexico and Guatemala.', ' A species of butterwort, it forms summer rosettes of flat, succulent leaves up to 10\\xa0centimeters (4\\xa0in) long, which are covered in mucilaginous (sticky) glands that attract, trap, and digest arthropod prey.', ' Nutrients derived from the prey are used to supplement the nutrient-poor substrate that the plant grows in.', ' In the winter the plant forms a non-carnivorous rosette of small, fleshy leaves that conserves energy while food and moisture supplies are low.', ' Single pink, purple, or violet flowers appear twice a year on upright stalks up to 25 centimeters long.']], ['Phylica', ['Phylica is a genus of plants in the family Rhamnaceae.', ' It contains about 150 species, the majority of which are restricted to South Africa, where they form part of the \"fynbos \".', ' A few species occur in other parts of southern Africa, and on islands including Madagascar, the Mascarene Islands, Île Amsterdam, Saint Helena, Tristan da Cunha, and Gough Island.']], ['Salvia chamelaeagnea', ['Salvia chamelaeagnea is a species of flowering plant in genus \"Salvia\", known as sages.', ' It is endemic to South Africa, where it grows on the western coastline of the Cape of Good Hope.', ' It is a shrubby perennial herb up to 6 ft tall and 4 ft wide.', ' It bears 3/4 in light violet-blue flowers with pale lower lips and white throats.', ' The small, green leaves release a slight medicinal odor when brushed.', ' In the wild, the plant grows in sandy soil in streambeds, open fields, and roadsides.', ' It is cultivated for gardens.']], ['Chorizanthe watsonii', ['Chorizanthe watsonii is a species of flowering plant in the buckwheat family known by the common name fivetooth spineflower.', ' It is native to the western United States from Washington to the Mojave Desert.', ' It grows in many types of plant communities from desert scrub to woodland and sagebrush.', ' This small plant grows a woolly erect stem up to about 15 centimeters tall.', ' The inflorescence is a cluster of flowers surrounded by five hairy greenish bracts tipped with hooked awns.', ' The flower is about 2 millimeters wide and yellow in color.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.664\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac1c73e55429964131be28a', 'answer': 'Country Thuggin', 'question': 'Faces is a mixtape that features guest appearances from an American rapper who released one album with Souf Folk called what?', 'supporting_facts': [['Faces (Mac Miller album)', 0], ['Faces (Mac Miller album)', 2], ['Mike Jones (rapper)', 0], ['Mike Jones (rapper)', 3]], 'context': [['Baby Face Killa', ['Baby Face Killa is the ninth mixtape by American rapper Freddie Gibbs.', ' It was released on September 25, 2012.', ' It is the first installment of the Gangsta Grillz series featuring Gary, Indiana based Freddie Gibbs.', ' The mixtape was announced shortly after the release of the previous street album \"Cold Day In Hell\".', ' There are 18 tracks on the street album and additional bonus tracks on the deluxe edition that were released on iTunes.', ' The mixtape features guest appearances from Pharrell Williams, Dana Williams, Z-Ro, Young Jeezy, Slick Pulla, Ea$y Money, YG, Dom Kennedy, SpaceGhostPurrp, Krayzie Bone, Jadakiss, Jay Rock, Wayne Blazed, Curren$y, Problem, G-Wiz, D-Edge, Hit Skrewface & Kirko Bangz.']], ['Dedication 5', ['Dedication 5 is a mixtape by American rapper Lil Wayne.', ' The mixtape was released on September 1, 2013.', ' It is the fifth installment of Lil Wayne\\'s \"\"Dedication\"\" series, following its predecessors \"The Dedication\", \"Dedication 2\", \"Dedication 3\" and \"Dedication 4\".', ' It\\'s also the fifth installment of Lil Wayne\\'s \"Gangsta Grillz\" Pentalogy.', ' The mixtape features guest appearances from The Weeknd, Chance the Rapper, Jae Millz, Future, Birdman, Mack Maine, T.I., Vado and Kidd Kidd among others.']], ['Stolen Youth (album)', ['Stolen Youth is a collaboration mixtape by American rapper Vince Staples and rapper/producer Mac Miller, under his production pseudonym Larry Fisherman.', ' The mixtape was released as a free digital download on June 20, 2013 to mixtape hosting websites.', ' \"Stolen Youth\" was entirely produced by Miller.', \" The mixtape features guest appearances from Mac Miller, Ab-Soul, Schoolboy Q, Da$H, Hardo, and Staples' Cutthroat Boyz cohort, Joey Fatts.\"]], ['Dreamchasers 3', ['Dreamchasers 3 is the eighth mixtape by American rapper Meek Mill.', ' It was released for free download on September 29, 2013, by Maybach Music Group and Dream Chasers Records.', ' The mixtape also released on mixtape hosting websites.', ' The mixtape features guest appearances from Travis Scott, Diddy, Nicki Minaj, Rick Ross, Mase, French Montana, Future, Yo Gotti, Fabolous and Jadakiss, along with his Dream Chasers Records artists Lil Snupe, Omelly, Louie V Gutta and Guordan Banks, among others.']], ['Mike Jones (rapper)', ['Michael Jones (born November 18, 1981) is an American rapper, actor, and entrepreneur.', ' He initially was with Swishahouse, then left to found his own label, Ice Age.', ' Before he was on Swishahouse he was in a group called Souf Folk, in which he used the alias Sache.', ' He released one album with Souf Folk called \"Country Thuggin\" in 2003.', ' He is also known for his catchphrase \"Mike Jones, who?\"', ' usually repeated several times and for handing out shirts with his cell phone number (281-330-8004) printed on the back.']], ['Dreamchasers 2', ['Dreamchasers 2 is a mixtape by American rapper Meek Mill (hosted by DJ Drama).', ' It was released on May 7, 2012, by Maybach Music Group and Warner Bros.', ' Records, and also released for digital download on DatPiff.', ' It serves as the second installment in the \"Dreamchasers\" series, following \"Dreamchasers\" (2011).', ' The mixtape features guest appearances from American rappers Travis $cott, Rick Ross, Fabolous, French Montana, Wale, Big Sean, Kendrick Lamar, Mac Miller, 2 Chainz and a Canadian rapper Drake – along with singing vocals by Jeremih, Trey Songz and Jordanne; as well as production that was provided by Jahlil Beats, SAP, All Star, Cardiak, Reginald Smith and Jesse Wilson, among others.', ' The mixtape consists mostly of original material, including a remix to Meek Mill\\'s single from his previous mixtape \"Dreamchasers\", \"House Party\", and a cover of Drake\\'s \"The Ride\".']], ['The Real Is Back', ['The Real Is Back is the tenth mixtape by American rapper Jeezy, It was released on May 28, 2011.', ' The Mixtape features guest appearances from 211, Slick Pulla, Scrilla, Fabolous, Yo Gotti, Lil Wayne, 2 Chainz, Boo, Freddie Gibbs, and Alley Boy. \"', 'Ballin\\'\" featuring Lil Wayne debuted and peaked at #57 on the \"Billboard\" Hot 100 and would eventually be included as a track on the deluxe edition of Jeezy\\'s fourth studio album \"\".', \" Since the mixtape's release, it has been downloaded over 100k times on DatPiff, certifying the mixtape gold.\"]], ['Faces (Mac Miller album)', ['Faces is the eleventh mixtape by American hip hop recording artist Mac Miller.', ' It was released online via free digital download on May 11, 2014, by REMember Music.', ' The mixtape features guest appearances from Rick Ross, Earl Sweatshirt, ScHoolboy Q, Mike Jones, Sir Michael Rocks, Vince Staples, Ab-Soul, Da$H, and King Ralph of Malibu.']], ['You Are Forgiven', ['You Are Forgiven is the debut mixtape by American hip hop recording artist MadeinTYO.', ' It was originally released on April 27, 2016 by Privateclub Records on DatPiff, then it was re-released on August 19, 2016 on iTunes by Privateclub Records and Warner Bros.', ' Records.', ' \"You Are Forgiven\" features production from MadeinTYO himself, along with K Swisha, Richie Souf, Purps and ICYTWAT.', ' It features guest appearances from 2 Chainz and Travis Scott.', ' The mixtape was supported by the singles \"Uber Everywhere\", and \"I Want\".']], ['OKE: Operation Kill Everything', ['OKE: Operation Kill Everything (simply known as OKE) is the fourteenth mixtape by American rapper The Game, which was hosted by DJ Skee.', ' The mixtape was released on October 8, 2013, and serves as his first project since leaving the Interscope Records, following the release of his fifth album \"Jesus Piece\" (2012).', ' It was also released in promotion of his upcoming sixth studio album.', ' The mixtape features guest appearances from Too Short, Schoolboy Q, Chris Brown, Lil Wayne, Problem, Nipsey Hussle, Juicy J, Young Jeezy, Clyde Carson, Skeme, Stat Quo and Ty Dolla $ign; as well as the production provided by Cardiak, DJ Mustard, SAP and Cool & Dre, among others.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.664\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a88982155429938390d3f81', 'answer': '1989', 'question': 'What year did the shopping centre on the street numbered B411 open in the space of in the retail space of the former William Whiteley Limited department store?', 'supporting_facts': [['Queensway, London', 1], ['Queensway, London', 3], ['Whiteleys', 0], ['Whiteleys', 1]], 'context': [['Metcalf South Shopping Center', ['Metcalf South Shopping Center was a shopping mall in Overland Park, Kansas.', ' It opened in 1967, near a large, unique department store called the French Market, which later became a strip mall anchored by Kmart and Hancock Fabrics (the Kmart closed in late 2013 and Hancock announced a move in early 2014).', ' The Metcalf South mall itself originally featured two main floors of retail space, although later a third floor of retail space was added, which in recent years became home to office space.', \" It featured two anchor stores (Sears and the Jones Store Company), later taken over by Macy's.\", \" Sears and the Glenwood Arts movie theater remained open in later years, while Macy's announced the closure of its Metcalf South store in January 2014.\"]], ['Westfield Stratford City', ['Westfield Stratford City is a shopping centre in Stratford, London.', ' The centre opened on 13 September 2011.', ' With a total retail floor area of 1905542 sqft , it is one of the largest urban shopping centres in Europe.', ' It is the third-largest shopping centre in the United Kingdom by retail space behind the MetroCentre and the Trafford Centre.', ' Taking the surrounding shopping area into account, it is the largest urban shopping centre in the European Union in terms of size.']], ['William Whiteley Limited', ['William Whiteley Limited was a large British retail company founded by William Whiteley in 1863.', ' The business grew to include four department stores and a warehouse removals business.']], ['Whiteleys', ['Whiteleys is a large shopping centre in Bayswater, London, England, which opened in 1989.', \" It has been built in the retail space of the former William Whiteley Limited department store, and opened in 1911 as London's first department store.\", \" The store's main entrance was located on Queensway.\"]], ['William Whiteley', ['William Whiteley (29 September 1831 – 24 January 1907) was an English entrepreneur of the late 19th and early 20th centuries.', ' He was the founder of the William Whiteley Limited retail company whose eponymous department store became the Whiteleys shopping centre.']], ['King Cross Jankomir', ['King Cross Jankomir is a shopping centre located in Zagreb, Croatia, on Velimir Škorpik Street 34 in the neighbourhood of Jankomir.', ' It was financed by Coimpredil and Coop Consumatori Nordest and opened on 21 September 2002.', ' The shopping centre has a gross area of 110000 m2 , 43000 m2 of which is covered, and 29000 m2 of retail space.', ' At the time of its opening, it was the largest shopping centre in Croatia.', ' The shopping centre cost €67\\xa0million to build, and its investors partly funded the upgrade of the adjoining road infrastructure.']], ['The Liberty', ['The Liberty is a covered shopping centre located in the London Borough of Havering.', ' It is the largest shopping centre in Romford.', ' It was originally built in 1968 as the Liberty Shopping Centre and underwent a four-year redevelopment completed in 2004.', ' The centre takes its name from the former Liberty of Havering and is owned by the Cosgrave Property Group.', ' It is the largest indoor shopping centre in the borough and covers 730000 sqm with 36400 sqm of retail space, around 100 shops.', ' The Liberty has an annual footfall of 23 million, equating to 425,000 people per week.', ' It is linked to Mercury Shopping Centre (formerly Liberty 2) by an underpass.']], ['Queensway, London', [\"Queensway (formerly Queen's Road) is a bustling cosmopolitan street in Bayswater, an area of west London.\", ' It is home to Whiteleys, many restaurants (especially Chinese and Middle Eastern), cafés, pubs, souvenir shops and a few high-street retail chains.', ' Queensway and Westbourne Grove are identified in the London Plan as one of 35 major centres in Greater London.', ' The street is numbered the B411 in the British road numbering scheme.']], ['Athlone Town Centre', ['Athlone Towncentre is a shopping centre located in Athlone, County Westmeath, Ireland.', ' The shopping centre is the largest shopping centre in the Irish midlands with over 140,000 sq meters of retail space consisting of 60 high end retail shops.', ' The Shopping centre is located in the heart of Athlone town enclosing a site on Dublin Gate Street and Gleeson Street.', ' The Shopping Centre opened in 2007.', ' The 4 star Sheraton Hotel adjoins the site and consists of 161 beds.', ' Anchor tenants include Marks and Spencer, River Island, Tommy Hilfiger, Topshop, Next, H&M, Monsoon amongst many more.']], ['Del Monte Center', ['Del Monte Center is an open-air shopping center located in Monterey, California.', ' Del Monte Center is the largest shopping center on the Monterey Peninsula and the second largest shopping mall in Monterey County, California, and has the only department store in a 22-mile radius.', ' Del Monte Center was designed by architect John Carl Wernecke, built by Williams and Burrows Construction Company and originally opened in 1967 but expanded and renovated in 1987.', \" The shopping center encompasses 675000 sqft of retail space including 85 stores, one department store (Macy's), Whole Foods Market, restaurants (California Pizza Kitchen, P.F. Chang's China Bistro, Pizza My Heart, Islands Fine Burgers & Drinks, Subway, Chipotle Mexican Grill, Starbucks and Lalla Grill), a gym and spa (Energia) and a thirteen screen Century Theatres.\", \" Petco was added in 2004, replacing Stroud's.\", \" The existing theater complex moved in 2006, with the former complex becoming a furniture store for Macy's.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.666\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac4e9ab5542996feb3fe974', 'answer': 'Roger Jason Stone Jr.', 'question': 'which American political consultant was a former Trump campaign ', 'supporting_facts': [['Jack Posobiec', 3], ['Roger Stone', 0]], 'context': [['Mary Matalin', ['Mary Joe Matalin (born August 19, 1953) is an American political consultant well known for her work with the Republican Party.', ' She has served under President Ronald Reagan, was campaign director for George H. W. Bush, was an assistant to President George W. Bush, and counselor to Vice President Dick Cheney until 2003.', ' Matalin has been chief editor of Threshold Editions, a conservative publishing imprint at Simon & Schuster, since March 2005.', ' She is married to Democratic political consultant James Carville.', ' She appears in the award-winning documentary film \"\" and also played herself, opposite her husband, James Carville, John Slattery, and Mary McCormack in the short lived HBO series \"K Street\".']], ['Rick Davis (politics)', ['Richard H. \"Rick\" Davis, Jr. (born 1957) is an American political consultant.', ' He currently serves as a Partner and Chief Operating Officer of Pegasus Sustainable Century Merchant Bank, a private equity firm specializing in sustainable development projects.', ' He is a managing partner of the business development and public affairs consulting firm Davis-Manafort, located in Alexandria, Virginia.', \" He is best known for being the National Campaign Manager of John McCain's 2008 Presidential campaign (from April 25, 2007 to November 4, 2008).\", ' In that capacity, he oversaw the development and implementation of all campaign strategy and policy development.', ' Davis also served McCain as National Campaign Manager for his 2000 Republican Presidential Primary campaign ( April 6, 1999 to March 9, 2000).']], ['George Birnbaum', ['George E. Birnbaum is an American international political consultant.', ' He was raised in Atlanta, Georgia, and has worked on dozens of United States Congressional and Senatorial races.', ' In 1998 he moved to Israel to serve as a consultant to Prime Minister Benjamin Netanyahu, became his chief of staff, and afterwards formed a partnership with political consultant Arthur Finkelstein.', ' His work includes polling, strategy, paid media and grassroots coalition building, developing and implementing campaign strategies.', ' During his career, George Birnbaum has worked on campaigns on 5 continents and has helped elect over 15 Presidents and Prime Ministers worldwide.']], ['Roger Stone', ['Roger Jason Stone Jr. (born August 27, 1952) is an American political consultant, lobbyist, and strategist, noted for his use of opposition research usually for candidates of the Republican Party.']], ['Basket of deplorables', ['\"Basket of deplorables\" is a phrase from a 2016 presidential election campaign speech delivered by Democratic nominee Hillary Clinton on September 9, 2016, at a campaign fundraising event, which Clinton used to describe a faction of supporters of her general election opponent, Republican nominee Donald Trump.', ' Clinton later said that she \"regrets saying half [of Trump\\'s supporters]\", and the Trump campaign repeatedly used the phrase against her during and after the 2016 presidential election.', ' Many Trump supporters adopted the \"Deplorable\" moniker for themselves.', \" After Clinton's loss, some journalists and political analysts questioned whether or not the speech played a role in the election's outcome.\"]], ['Fred Karger', ['Fred S. Karger (born January 31, 1950) is an American political consultant, gay rights activist and watchdog, former actor, and politician.', ' His unsuccessful candidacy for the Republican nomination for the 2012 US Presidential election made him the first openly gay presidential candidate in a major political party in American history.', ' Although he has not held elected or public office, Karger has worked on nine presidential campaigns and served as a senior consultant to the campaigns of Presidents Ronald Reagan, George H. W. Bush and Gerald Ford.', ' Karger was a partner at the Dolphin Group, a California campaign consulting firm.', \" He retired after 27 years and has since worked as an activist on gay rights causes, from protecting the gay bar The Boom to using his organization Californians Against Hate to investigate The Church of Jesus Christ of Latter-day Saints (LDS Church) and the National Organization for Marriage's campaigns to repeal the state's same-sex marriage law.\"]], ['Jack Posobiec', ['Jack Posobiec ( ) is an American alt-right pro-Donald Trump Internet activist and conspiracy theorist, known primarily for his controversial comments on Twitter.', ' During the 2016 election, he was a special projects director of Citizens for Trump, a pro-Trump organization.', ' For two months in 2017, he was a correspondent for \"The Rebel\", a far-right Canada-based website.', ' He was granted press access to the White House in April 2017, and his tweets have been promoted by former Trump campaign manager Roger Stone.']], ['Dick Morris', ['Richard Samuel \"Dick\" Morris (born November 28, 1946) is an American political author and commentator who previously worked as a pollster, political campaign consultant, and general political consultant.']], ['Joseph Napolitan', ['Joseph Napolitan (March 6, 1929 – December 2, 2013) was an American political consultant, who worked as a general consultant on over 100 political campaigns in the United States, and many others throughout the world.', ' Napolitan served on the 1960 Kennedy for President campaign, was Director of Media for the 1968 Hubert Humphrey campaign, and received the French Legion of Honour in 2005.', ' He died on December 2, 2013 at the age of 84.']], ['Roger Ailes', ['Roger Eugene Ailes (May 15, 1940\\xa0– May 18, 2017) was an American television executive and media consultant.', ' He was the founder and one-time Chairman and CEO of Fox News and the Fox Television Stations Group, from which he resigned in July 2016 following allegations that he sexually harassed female colleagues.', \" Ailes was a media consultant for Republican presidents Richard Nixon, Ronald Reagan, and George H. W. Bush, and for Rudy Giuliani's first mayoral campaign.\", ' In 2016, after he left Fox News, he became an adviser to the Donald Trump campaign, where he assisted with debate preparation.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.666\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac017945542997d642959bc', 'answer': '\"Hacksaw Ridge\"', 'question': 'What film did the sound editor for a 2013 Hong Kong-Chinese martial arts drama film based on the life story of the Wing Chun grandmaster Ip Man win two Academy Awards for?', 'supporting_facts': [['Robert Mackenzie (sound engineer)', 1], ['The Grandmaster (film)', 0]], 'context': [['The Grandmaster (film)', ['The Grandmaster is a 2013 Hong Kong-Chinese martial arts drama film based on the life story of the Wing Chun grandmaster Ip Man.', ' The film was directed and written by Wong Kar-wai and stars Tony Leung as Ip Man.', ' It was released on 8 January 2013 in China.', ' It was the opening film at the 63rd Berlin International Film Festival in February 2013.', ' The film was selected as part of the 2013 Hong Kong International Film Festival.', ' The Weinstein Company acquired the international distribution rights for the film.', ' The film was selected as the Hong Kong entry for the Best Foreign Language Film at the 86th Academy Awards, making the January shortlist, but did not get the nomination.', ' The film was nominated for Best Cinematography (Philippe Le Sourd) and Best Costume Design (William Chang Suk Ping) at the 86th Academy Awards.']], ['Ip Man: The Final Fight', ['Ip Man: The Final Fight is a 2013 Hong Kong biographical martial arts film directed by Herman Yau, starring Anthony Wong, Anita Yuen, Jordan Chan, Eric Tsang and Gillian Chung.', ' It is based on the life of the Wing Chun grandmaster Ip Man.', ' This film has no connection to the earlier Ip Man films, such as Wilson Yip\\'s \"Ip Man\" and \"Ip Man 2\", and Wong Kar-wai\\'s \"The Grandmaster\".']], ['Ip Man 2', ['Ip Man 2 (also known as Ip Man 2: Legend of the Grandmaster) is a 2010 Hong Kong biographical martial arts film loosely based on the life of Ip Man, a grandmaster of the martial art Wing Chun.', ' A sequel to the 2008 film \"Ip Man\", \"Ip Man 2\" was directed by Wilson Yip and stars Donnie Yen, who reprises the leading role.', \" Continuing after the events of the earlier film, the sequel centers on Ip's movements in Hong Kong, which is under British colonial rule.\", ' He attempts to propagate his discipline of Wing Chun, but faces rivalry from other practitioners, including the local master of Hung Ga martial arts.']], ['Ip Man (TV series)', ['Ip Man is a 2013 Chinese television series romanticising the life of Ip Man (Mandarin: Ye Wen), a Chinese martial artist specialising in Wing Chun.', ' Directed by Fan Xiaotian, the series starred Hong Kong actor Kevin Cheng as the title character, with Han Xue, Liu Xiaofeng, Chrissie Chau, Song Yang, Yu Rongguang, Yuen Wah and Bruce Leung as part of the supporting cast.', ' Wilson Yip, the director of the films \"Ip Man\" and \"Ip Man 2\" (starring Donnie Yen), and Taiwanese producer Young Pei-pei served as the artistic consultants for the series, while Ip Man\\'s sons, Ip Chun and Ip Ching, served as the martial arts consultants.', ' The series was shot from July–November 2012 in Kunshan, Suzhou, and was first aired on Shandong TV from 24 February to 9 March 2013.', ' It won the Golden Eagle Award for Best Television Series in 2012.']], ['Ip Man (film)', ['Ip Man is a 2008 Hong Kong biographical martial arts film based on the life of Ip Man, a grandmaster of the martial art Wing Chun and teacher of Bruce Lee.', \" The film focuses on events in Ip's life that supposedly took place in the city of Foshan during the Sino-Japanese War.\", ' The film was directed by Wilson Yip, and stars Donnie Yen as Ip Man, with martial arts choreography by Sammo Hung.', ' The supporting cast includes Simon Yam, Lynn Hung, Lam Ka-tung, Xing Yu, Hiroyuki Ikeuchi and Tenma Shibuya.']], ['Dennis To', ['Dennis To Yu-hang (born 1 January 1981) is a Hong Kong martial artist and actor.', ' He started his career as a wushu practitioner and won several awards at various competitions, including a silver medal at the 2002 Asian Games and a gold medal at the 2005 East Asian Games.', ' To became an actor in 2007 and started off by playing minor roles in \"Ip Man\" (2008), \"Bodyguards and Assassins\" (2009) and \"Ip Man 2\" (2010).', ' He is best known for his role as the Wing Chun grandmaster Ip Man in the 2010 film \"The Legend Is Born – Ip Man\".']], ['Ip Man (film series)', ['Ip Man is a series of Hong Kong biographical martial arts films starting with \"Ip Man\" in 2008 and followed by two sequels – \"Ip Man 2\" (2010) and \"Ip Man 3\" (2015).', ' All three films are directed by Wilson Yip, written by Edmond Wong, produced by Raymond Wong and star Donnie Yen.', ' Mandarin Films released the first two films in Hong Kong, which earned more than $37 million with a budget of around $24.6 million.', ' The films are based on the life events of the Wing Chun master of the same name.', ' Donnie Yen has mentioned each film has a unique theme, that the first \"Ip Man\" film was about \"Survival\", \"Ip Man 2\" focuses on \"Making a Living and Adaptation\", while \"Ip Man 3\" focuses on \"Life\" itself.']], ['Robert Mackenzie (sound engineer)', ['Robert Mackenzie is an Australian supervising sound editor.', ' He is best known for his work on \"Animal Kingdom\" (2010), \"The Hunter\" (2011), \"Lore\" (2012), \"Felony\" (2013), \"The Grandmaster\" (2013), \"The Rover\" (2014), \"Deadline Gallipoli\" (2015), \"Partisan\" (2015), \"Lion\", and critically acclaimed war-drama film \"Hacksaw Ridge\", for which he received two Academy Award nominations at the 89th Academy Awards, Best Sound Editing and Best Sound Mixing .']], ['The Legend Is Born: Ip Man', ['The Legend Is Born – Ip Man is a 2010 Hong Kong biographical martial arts film based on the early life of the Wing Chun grandmaster Ip Man, directed by Herman Yau and starring Dennis To in the titular role.', ' Though not made in collaboration with Wilson Yip\\'s \"Ip Man\" or \"Ip Man 2\", \"The Legend is Born\" features several actors who appeared in Yip\\'s films, including Sammo Hung, Louis Fan, and Chen Zhihui.', ' The film also features a special appearance by Ip Chun, the son of Ip Man.']], ['Ip Man 3', ['Ip Man 3 is a 2015 Hong Kong biographical martial arts film directed by Wilson Yip, produced by Raymond Wong and written by Edmond Wong with action choreography by Yuen Woo-ping.', ' It is the third in the \"Ip Man\" film series based on the life of the Wing Chun grandmaster Ip Man and features Donnie Yen reprising the title role.', \" The film also stars Mike Tyson, and Yip Man's pupil Bruce Lee is portrayed by Danny Chan.\", ' Principal photography commenced in March 2015 and ended in June that year.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.667\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a74106b55429979e288289e', 'answer': 'Mumbai', 'question': 'Where is the company that Sachin Warrier worked for as a software engineer headquartered? ', 'supporting_facts': [['Sachin Warrier', 3], ['Tata Consultancy Services', 0]], 'context': [['Lead programmer', ['A lead programmer is a software engineer in charge of one or more software projects.', ' Alternative titles include \"development lead\", \"technical lead\", \"lead software engineer\", \"software design engineer lead\" (SDE lead), \"software development manager\", \"software manager\", or \"lead application developer\".', ' When primarily contributing in a high-level enterprise software design role, the title \"software architect\" (or similar) is often used.']], ['Sachin Bansal', [\"Sachin Bansal (born 5 August 1981) is an Indian Software engineer and Internet entrepreneur known for co-founding India's e-commerce platform Flipkart.\", ' Sachin is from Chandigarh and is an engineering graduate from Indian Institute of Technology Delhi.']], ['William Connolley', ['William Michael Connolley (born 12 April 1964) is a British software engineer, writer, and blogger on climatology.', ' Until December 2007 he was Senior Scientific Officer in the Physical Sciences Division in the Antarctic Climate and the Earth System project at the British Antarctic Survey, where he worked as a climate modeller.', ' After this he became a software engineer for Cambridge Silicon Radio.']], ['Wes McKinney', ['Wes McKinney is an American statistician, data scientist and businessman.', ' He was the CEO and founder of technology startup Datapad.', ' He is the main author of the open-source Pandas package for data analysis in the Python programming language, and has also written the textbook \"Python for Data Analysis\" on the topic.', ' He worked as a software engineer for Cloudera following their acquisition of Datapad in 2014.', ' He is now a software engineer at Two Sigma Investments.']], ['Sachin Warrier', ['Sachin Warrier is a playback singer and composer in the Malayalam cinema industry from Kerala.', ' He became notable with the song \"Muthuchippi Poloru\" from the film Thattathin Marayathu.', ' He made his debut with the movie Malarvaadi Arts Club.', ' He was working as a software engineer in Tata Consultancy Services in Kochi.', ' Later he resigned from the job to concentrate more on music.', ' His latest work is as a composer for the movie Aanandam.']], ['Muthuchippi Poloru', ['\"Muthuchippy Poloru\" is a Malayalam song composed by Shaan Rahman that featured in the film \"Thattathin Marayathu\".', ' Written by Anu Elizabeth Jose.', ' It was sung by Sachin Warrier and actress Remya Nambeesan.', ' The song was one of the most popular Malayalam songs of 2012 and won several awards.', ' It was later reused in the Telugu remake of \"Thattathin Marayathu\", \"Saheba Subramanyam\", as \"Muddu Muddu\".']], ['Tata Consultancy Services', ['Tata Consultancy Services Limited (TCS) is an Indian multinational information technology (IT) service, consulting and business solutions company Headquartered in Mumbai, Maharashtra.', ' It is a subsidiary of the Tata Group and operates in 46 countries.']], ['Kanave Kalayathe', ['Kanave Kalayathe (English: Dreams Never Disappear ) is an album sung by Sachin Warrier starring Leon Poulose, Swetha Raj, Akash Thomas, NeethuKrishna VR, Christy Vazhapilly and Althaf.', ' The soundtrack was composed by Abee Joe.', ' It was produced by Nidhinsha and distriburted by Muzik247.', ' The music video is directed by Deen Shifaz and Ashith wilson.']], ['Alec Muffett', ['Alec David Edward Muffett (born April 22, 1968) is an Anglo-American internet-security evangelist, architect, and software engineer.', ' He is principally known for his work on Crack, the original Unix password cracker, and for the CrackLib password-integrity testing library; he is also active in the Open Source software community.', ' He worked as a Software Engineer for Facebook.']], ['Divya S. Menon', ['Divya S. Menon (born 14 March 1987) is an Indian singer and television anchor from Kerala.', ' Divya is a playback singer in Malayalam who has also recorded songs for Tamil and Telugu films.', \" Divya started anchoring musical shows in Asianet Cable Vision (Thrissur) and has done musical shows in YesIndiavision(Mementos) and Kairali Channels(Ganamela, Sing 'N' Win and Rain drops).\", ' She started her film career with Ee Pattanathil Bhootham.', ' She was noticed by Shaan Rahman while anchoring music shows and picked her for Vineeth Sreenivasan - Shaan Rahman debut album, Coffee @ MG Road.', ' She is associated with Blogswara and have sung in multiple albums in the series.', \" Divya has sung in Vineeth Sreenivasan's super hit romantic movie, Thattathin Marayathu composed by Shaan Rahman.\", ' She has been associated with Vineeth - Shaan ventures, including Malarvadi Arts Club.', ' In 2014 the hit wedding song \"Thudakkham Maangalyam\" from Anjali Menon\\'s Bangalore Days gave her much popularity which she sang along with along with Vijay Yesudas and Sachin Warrier composed by Gopi Sunder.', ' In 2015 Divya was noticed more promptly when she sang the song \"Puthumazhayai\" from Martin Prakkat\\'s Charlie (2015 Malayalam film) composed by Gopi Sundar while the same song was sung by Shreya Ghoshal too.', ' She also sang several ad jingles for various music composers both in Malayalam and Tamil.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.668\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abea05f5542991f661061b6', 'answer': 'Biscayne National Park to the east and Everglades National Park to the west', 'question': 'South Dade High School is located between what two national parks?', 'supporting_facts': [['South Dade High School', 0], ['Homestead, Florida', 0]], 'context': [['Miami Northwestern Senior High School', ['Miami Northwestern Senior High School is a public 4-year high school located in Miami, Florida, United States, serving students in grades 9-12 from the Liberty City neighborhood of Miami.', ' The school colors are old gold and royal blue.', ' The average annual enrollment is approximately 1,800 students.', ' Miami Northwestern was founded in 1955 to serve the increasing population of northern Miami.', \" Shortly after the school's inception, the Bull was chosen as the official school mascot from the former Dorsey High School.\", ' Miami Northwestern originally served as an all-black high school.', ' Beginning in 1966, Dade County high schools stopped being segregated, and most students from Booker T. Washington transferred to Northwestern (and Miami Jackson Senior High School) in 1967–1968.']], ['Yala National Park', ['Yala National Park is the most visited and second largest national park in Sri Lanka.', ' The park consists of five blocks, two of which are now open to the public, and also adjoining parks.', \" The blocks have individual names such as, Ruhuna National Park (block 1) and Kumana National Park or 'Yala East' for the adjoining area.\", ' It is situated in the southeast region of the country, and lies in Southern Province and Uva Province.', ' The park covers 979 km2 and is located about 300 km from Colombo.', ' Yala was designated as a wildlife sanctuary in 1900, and, along with Wilpattu was one of the first two national parks in Sri Lanka, having been designated in 1938.', ' The park is best known for its variety of wild animals.', ' It is important for the conservation of Sri Lankan elephants, Sri Lankan leopards and aquatic birds.']], ['Australian Alps National Parks and Reserves', ['The Australian Alps National Parks and Reserves is a group of eleven protected areas consisting of national parks, nature reserves and one wilderness park located in the Australian Capital Territory, New South Wales and Victoria and which was listed as a \"place\" on the Australian National Heritage List on 7 November 2008 under the \"Environment Protection and Biodiversity Conservation Act 1999\".', ' The listing which covers an area of 1653180 ha , contains the vast majority of alpine and sub-alpine environments in Australia.', ' The listing includes the following protected areas - Alpine, Baw Baw, Brindabella, Kosciuszko, Mount Buffalo, Namadgi and Snowy River national parks; the Avon Wilderness Park, and the Bimberi, Scabby Range and Tidbinbilla nature reserves.']], ['List of U.S. National Parks by elevation', ['This is a list of United States National Parks by elevation.', \" Most of America's national parks are located in mountainous areas.\", ' Even among those located close to the ocean, not all are flat.', ' Those few that are low-lying preserve important natural habitats that could never exist at high altitude.', ' Several national parks protect deep canyons with great vertical relief.', ' There are also three national parks whose primary features are caves, the depths of which are still being explored.']], ['High Sierra Camps', [\"The High Sierra Camps are nine rustic lodging facilities located in two national parks and a national monument in California's Sierra Nevada mountain range.\", ' Open most years from June or July to September, they are staffed camps with tent cabins and food service facilities.', ' The backcountry camps receive their supplies by pack mules.']], ['National parks of Scotland', ['National parks of Scotland are managed areas of outstanding landscape where habitation and commercial activities are restricted.', ' At present, Scotland has two national parks: Loch Lomond and The Trossachs National Park, created in 2002, and the Cairngorms National Park, created in 2003.', ' These were designated as such under the National Parks (Scotland) Act 2000 which was an early piece of legislation passed by the Scottish Parliament not long after its creation in 1999.', ' Scottish-born John Muir spearheaded the effort to create Yosemite National Park in the US, as well as the conservation movement at large.']], ['Homestead, Florida', ['Homestead is a city within Miami-Dade County in the U.S. state of Florida, between Biscayne National Park to the east and Everglades National Park to the west.', ' Homestead is primarily a Miami suburb and a major agricultural area.', ' It is a principal city of the Miami metropolitan area, which was home to an estimated 6,012,331 people at the 2015 census.']], ['Australian Alps Walking Track', ['The Australian Alps Walking Track is a long distance walking trail through the alpine areas of Victoria, New South Wales and ACT.', ' It is 655\\xa0km long, starting at Walhalla, Victoria and running through to Tharwa, ACT near Canberra.', ' The track weaves mainly through Australian national parks, such as Alpine National Park and Kosciuszko National Park, though it is not exclusively restricted to national parks.', ' It ascends many peaks including Mount Kosciuszko, Mount Bogong, and Bimberi Peak, the highest points in N.S.W., Victoria, and the A.C.T. respectively.', ' The AAWT crosses exposed high plains including the Victorian Bogong High Plains and the Main Range in NSW.', ' To walk the whole trail can take between 5 and 8 weeks.', ' Food drops or a support crew are necessary, as the trail passes through no towns, although it passes close to the ski resorts of Mt Hotham, Falls Creek, Mt Baw Baw, Thredbo, Charlotte Pass and Perisher.']], ['Yuraygir National Park', ['Yuraygir is a national park in New South Wales, Australia, located 482 km northeast of Sydney.', ' It was created in 1980, a result of the merger and enlargement of two national parks, Angourie and Red Rock National Parks, both of which had been established in 1975.', ' The name is a phonetic translation of the local indigenous tribe who had lived in the area, and had formerly been transcribed variously as Jeigir, Jiegera, Jungai, Yagir, Yegera, Yegir, Yiegera or Youngai.', ' At the time of its establishment in 1980, the park was fragmented, and parcels of land were bought over the following two decades to unite segments into a more contiguous protected area.', ' Sometimes these acquisitions required protracted negotiations (and legal disputes) with land owners.']], ['South Dade High School', ['South Dade Senior High School is a secondary school located in unincorporated Miami-Dade County, Florida, near Homestead.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.669\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac370ce5542995ef918c18c', 'answer': 'no', 'question': 'Are Southwest Florida International Airport and Henry E. Rohlsen Airport both in Florida?', 'supporting_facts': [['Southwest Florida International Airport', 0], ['Henry E. Rohlsen Airport', 0]], 'context': [['Anguilla, U.S. Virgin Islands', ['Anguilla is a settlement on the island of Saint Croix in the United States Virgin Islands.', ' It is located immediately east of Henry E. Rohlsen Airport.']], ['Hummingbird Air', ['Hummingbird Air is an airline offering scheduled and chartered air taxi services as well as cargo flights in the Caribbean.', ' It is based at Henry E. Rohlsen Airport on the island of St. Croix, U.S. Virgin Islands.', ' The airline was founded in late 2013 by Sam Raphael, a Dominican hotelier who wanted to improve air service to Dominica.', ' As of May 2016, Hummingbird Air serves five destinations in the Caribbean with a fleet of three Beechcraft Model 99 aircraft.']], ['U.S. Virgin Islands Highway 66', [\"U.S. Virgin Islands Highway 66 is a major east—west arterial on St. Croix in the United States Virgin Islands and is named the Melvin H. Evans Highway in honor of the territory's first elected governor.\", ' It is one of the few divided highways in a territory with the distinction of being the only US jurisdiction to drive on the left.', ' It is also the fastest road on the island—and in the territory—with a 55\\xa0mph speed limit for passenger vehicles (except buses) and a 40 mi/h limit for heavy trucks and buses.', ' All junctions are at-grade, there are stoplights with connecting roads, and driveway access is limited by default in the territory but the highway is not explicitly classified as an expressway.', \" It is an important intermediate link between locations on the southern coast, including an oil refinery of the Hess Corporation and the Henry E. Rohlsen International Airport, and the island's principal towns Christiansted and Frederiksted.\", ' The highway travels through areas of mostly lighter development and has multiple spurs and other intersections connecting to parallel roads and smaller communities.', ' St. Croix has no single encircling route so the highway is one of the most heavily used.']], ['Southwest Florida International Airport', ['Southwest Florida International Airport (IATA: RSW, ICAO: KRSW, FAA LID: RSW) is a county-owned airport in the South Fort Myers region of unincorporated Lee County, Florida.', \" The airport's market is Southwest Florida: Bonita Springs, Cape Coral, Captiva Island, Estero, Fort Myers, Marco Island, Naples and Sanibel Island.\", ' In 2015 passengers numbered 8,371,801.', ' The airport is the second busiest single-runway airport in the United States after San Diego International Airport.', ' It is a U.S. Customs and Border Protection port of entry.']], ['Capt. Henry E. Sewall House', ['The Capt. Henry E. Sewall House is an historic wooden house now located in Indian RiverSide Park in Jensen Beach, in Martin County, Florida.', \" Local pioneer and developer Capt. Henry E. Sewall built it in 1889 at the southern tip of Sewall's Point, the peninsula and town which bear his family name.\", \" When Capt. Sewall became the postmaster of Sewall's Point, the house served also as the Sewall's Point post office.\", ' Next to the house Capt. Sewall also built a freight dock that extended into the St. Lucie River along with a storage building at its end.']], ['Henry E. Rohlsen Airport', ['Henry E. Rohlsen Airport (IATA: STX, ICAO: TISX, FAA LID: STX) is a public airport located six miles (10 km) southwest of Christiansted on the island of St. Croix in the United States Virgin Islands.', ' The airport is named after Henry E. Rohlsen, a St. Croix native who was one of the Tuskegee Airmen during World War II.']], ['Gateway, Florida', ['Gateway is a census designated place (CDP) in Lee County, Florida, United States.', ' The population was 2,943 at the 2000 census.', ' It is part of the Cape Coral-Fort Myers, Florida Metropolitan Statistical Area.', ' The community is located just north of Southwest Florida International Airport.']], ['Tallahassee International Airport', ['Tallahassee International Airport (IATA: TLH,\\xa0ICAO: KTLH,\\xa0FAA LID: TLH) is a city-owned airport five miles southwest of downtown Tallahassee, in Leon County, Florida.', ' It serves the state capital of Florida, and its surrounding areas; it is one of the major airports in north Florida, the others being Pensacola International Airport, Northwest Florida Beaches International Airport, and Jacksonville International Airport.']], ['Florida State Road 876', ['Daniels Parkway, originally known as State Road\\xa0876 (SR\\xa0876), runs from Tamiami Trail (U.S. Highway\\xa041, US 41) in Fort Myers to SR\\xa082 in Lehigh Acres.', ' FDOT still maintains a small strip of road only slightly more than 1/2 mi long.', ' SR\\xa0876 intersects Interstate\\xa075 (I-75) and serves a nearby rest stop.', ' Daniels Parkway is the main artery serving Southwest Florida International Airport.']], ['Fort Myers, Florida', ['Fort Myers is the county seat and commercial center of Lee County, Florida, United States.', ' As of the 2010 census, the city population was 62,298 and in 2016 an estimate of 77,146.', ' Fort Myers is a gateway to the Southwest Florida region and a major tourist destination within Florida.', ' The winter homes of Thomas Edison (\"Seminole Lodge\") and Henry Ford (\"The Mangoes\") are a primary tourist attraction in the region.', ' The city is named after Colonel Abraham Myers.', ' The geographic statistical area is serviced by Southwest Florida International Airport (RSW), located southeast of the city.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.669\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abd74c95542993062266ca2', 'answer': 'Premier League club', 'question': 'Who does the current PFA Young Player of the Year currently play for?', 'supporting_facts': [['PFA Young Player of the Year', 3], ['Dele Alli', 0]], 'context': [['PFAI Young Player of the Year', [\"The Professional Footballers' Association of Ireland Players' Young Player of the Year (often called the PFAI Players' Young Player of the Year, the PFAI Young Player of the Year, or simply the Young Player of the Year) award is given to the footballer in the top-flight of Irish football, the League of Ireland, who is seen to have been the best player of the previous season and is under 23 years of age.\"]], ['Graeme Payne', ['Graeme Payne (born 13 February 1956 in Dundee) is a Scottish former footballer who played as a winger.', ' At Dundee United he played in two Scottish league cup final winning teams.', ' He was the first winner of the Scottish PFA Young Player of the Year award.']], ['SFWA Young Player of the Year', [\"The Scottish Football Writers' Association Young Player of the Year (often called the SFWA Young Player of the Year, or simply the Scottish Young Player of the Year) award is given to the footballer in the Scottish football league system, who is seen to have been the best young (under 23) player of the previous season.\", \" The shortlist is compiled by the members of the Scottish Football Writers' Association (the SFWA), who also vote for the winner.\", ' The prize is seen as the highest awarded to a young player as it names the \"Young Player of the Year\"; the footballer who is seen to have been \"the\" best young player over the previous season.', ' The award was first made in 2002, and was won by Motherwell forward James McFadden.']], ['PFA Young Player of the Year', [\"The Professional Footballers' Association Young Player of the Year (often called the PFA Young Player of the Year, or simply the Young Player of the Year) is an annual award given to the player aged 23 or under at the start of the season who is adjudged to have been the best of the season in English football.\", \" The award has been presented since the 1973–74 season and the winner is chosen by a vote amongst the members of the players' trade union, the Professional Footballers' Association (PFA).\", ' The first winner of the award was Ipswich Town defender Kevin Beattie.', ' The current holder is Dele Alli, who won the award for his performances throughout the 2016–17 campaign for Tottenham Hotspur.']], [\"PFA Women's Young Player of the Year\", [\"The Professional Footballers' Association Women's Young Player of the Year (commonly referred to as PFA Young Player of the Year) is an annual award given to the player who is voted to have been the best of the year in English women's football.\", \" The award has been presented since the 2013–14 season and the winner is chosen by a vote amongst the members of the players' trade union, the Professional Footballers' Association (PFA).\"]], [\"Phil O'Donnell (footballer)\", [\"Phillip O'Donnell (25 March 1972 – 29 December 2007) was a Scottish footballer, who played as a left-sided midfielder for Motherwell, Celtic and Sheffield Wednesday during his career.\", ' He also earned one international cap for Scotland, and twice won the Scottish PFA Young Player of the Year award.', ' He died after suffering cardiac arrest while playing for Motherwell against Dundee United on 29 December 2007.']], ['Harry Kewell', ['Harry Kewell (born 22 September 1978) is an Australian football coach and former player who is the head coach of League Two club Crawley Town.', ' Kewell played for Leeds United, Liverpool, Galatasaray, Melbourne Victory, Al-Gharafa and Melbourne Heart.', ' While at Leeds he was named the PFA Young Player of the Year in 2000.', ' Internationally he has received 58 caps, and scored 17 goals, while playing for the Australian national team.', ' A left winger also capable of playing as an attacking midfielder or second striker, he is often regarded within the media as \"Australia\\'s finest football export\", despite his career being blighted with injury.', \" In 2012, Kewell was named Australia's greatest footballer in a vote by Australian fans, players and media.\"]], ['PFA Scotland Young Player of the Year', ['The PFA Scotland Young Player of the Year, formerly known as the Scottish PFA Young Player of the Year, is named at the end of every Scottish football season.', \" The members of the Professional Footballers' Association Scotland vote on which of its young members played the best football in the previous year.\", ' The award was first given in 1978, to Graeme Payne.', ' The Bulgarian international Stiliyan Petrov was the first non-Scottish player to win the award, when he did so in 2001.']], ['1984–85 Manchester United F.C. season', [\"The 1984–85 season was Manchester United's 83rd season in the Football League, and their 10th consecutive season in the top division of English football.\", ' They defeated Everton 1–0 in the FA Cup Final to win the trophy for the sixth time, and finished fourth in the league.', ' It was the first season at the club for new signings Gordon Strachan, Jesper Olsen and Alan Brazil, while Mark Hughes became established in the forward line alongside Frank Stapleton, with Norman Whiteside moving into central midfield to replace the departed Ray Wilkins.', \" Hughes ended the season as United's top scorer with 24 goals (16 in the league) and was also voted PFA Young Player of the Year.\", \" Brazil, however, failed to establish himself as a regular player, with Atkinson alternating between him and Frank Stapleton as the club's second striker to play alongside the prolific Hughes.\"]], ['Dele Alli', ['Bamidele Jermaine Alli (born 11 April 1996) is an English professional footballer who plays as a midfielder for Premier League club Tottenham Hotspur and the England national team.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.670\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac46bb05542991943173983', 'answer': 'no', 'question': 'Are both Schweizerischer Niederlaufhund and Portuguese Podengo dog breeds from Portugal?', 'supporting_facts': [['Schweizerischer Niederlaufhund', 0], ['Portuguese Podengo', 0]], 'context': [['Conformation (dog)', [\"Conformation in dogs refers solely to the externally visible details of a dog's structure and appearance, as defined in detail by each dog breed's written breed standard.\", ' A dog that \"conforms\" to most of the items of description in its individual breed standard is said to have \"good conformation\".', ' Unlike equine conformation, there are no fixed rules for dog conformation, as dogs are the most variable in appearance of any animals (\"Phenotypic variation among dog breeds, whether it be in size, shape, or behavior, is greater than for any other animal\").', ' Instead, conformation in dogs is based on the dog type from which the breed developed, along with many details that have been added to the breed standard for purposes of differentiation from other breeds, for working reasons, or for enhancing the beauty of the animals from the viewpoint of the fanciers who wrote the breed standards.']], ['Dog breed', ['Dog breeds are dogs that have relatively uniform physical characteristics developed under controlled conditions by humans, with breeding animals selected for phenotypic traits such as size, coat color, structure, and behavior.', ' The Fédération Cynologique Internationale recognizes over 400 pure dog breeds.']], ['Tugou', ['Tugou (土狗, pinyin: \"tǔ gǒu\"), literally means Native Dog in Mandarin Chinese, is the general name for several dog breeds originated from China and still abundantly exists across the country today.', ' Tugou includes the most popular Chinese dog breed - the Chinese Field Dog (, pinyin: \"zhōng huá tián yuán quǎn\"), Chinese Chongqing Dog, Xiasi Dog, and several other native dog breeds distributed across China.', ' They are roughly 45–50\\xa0cm tall at the shoulder.']], ['European Portuguese', ['European Portuguese (Portuguese: \"português europeu\" , ] ), also known as Lusitanian Portuguese (\"português lusitano \") and Portuguese of Portugal (\"português de Portugal \") in Brazil, refers to the Portuguese language spoken in Portugal.', ' Standard Portuguese pronunciation, the prestige norm based on European Portuguese, is the reference for Portugal, the Portuguese-speaking African countries, East Timor and Macau.', ' The word “European” was chosen to avoid the clash of “Portuguese Portuguese” (“\"português português \"”) as opposed to Brazilian Portuguese.']], ['Breed group (dog)', ['A breed group is a categorization of related breeds of animal by an overseer organization, used to organize the showing of animals.', ' In dogs, kennel clubs define the \"Breed Groups\" and decide which dog breeds are to be included in each breed group.', ' The Fédération Cynologique Internationale breed groups are used to organize dogs for international competition.', ' Breed groups often have the names of, and are loosely based on, ancestral dog types of modern dog breeds.']], ['Andalusian Hound', ['The Andalusian hound (Spanish: \"Podenco andaluz\" ) is a dog breed originating in Spain, especially Andalusia.', ' These dogs are similar to other Iberian breeds such as the Ibizan Hound, the Portuguese Podengo, the Podenco Canario and the Maneto.', ' In the Iberian Peninsula there are cave paintings representing dogs with a strong resemblance to these races.', \" Dogs very similar to these, including the Cirneco dell'Etna and Pharaoh Hound, have been bred in much of the Mediterranean basin since ancient times.\", ' Despite the widespread belief that the podencos were introduced into Spain some 3,000 years ago by the Phoenicians, recent genetic studies have concluded that these dogs actually have a close genetic relationship with other European hunting dogs and are no more \"primitive\" than the others.']], ['Romanian Raven Shepherd Dog', ['The Romanian Raven Shepherd Dog is a very large Romanian livestock guardian dog, taxonomized within the second group of dog breeds - Pinscher and Schanuzer - Molossoid Breeds - Swiss Mountain and Cattle Dogs, section 2.2: Molossoid Breeds - Mountain type.']], ['Schweizerischer Niederlaufhund', ['The Schweizerischer Niederlaufhund (FCI No, 60) (translated into English as the Small Swiss Hound, is a breed of dog of the scenthound type from Switzerland.', ' \"Niederlaufhund\" means short-legged hound.', ' The breed has a number of different varieties (all of the same breed).']], ['Portuguese Podengo', ['The Portuguese Podengo is an ancient multi-sensory hound (sight and scent) breed of dog from Portugal.', ' As a breed, the Podengo is divided into three size categories that are not interbred: small (Pequeno), medium (Medio) and large (Grande).', \" Their coats are either short and 'smooth', or longer and 'wired'.\", ' The smooth coated variety is traditional, dating back to the 5th century, whereas the wire coated variety is an outcome of the assimilation of various other breeds during the 20th century.', ' In general, the breed is healthy; the Pequeno (small) variety has an average lifespan of approximately 15–17 years.']], ['Vulnerable Native Breeds', ['Vulnerable Native Breeds are a group of dog breeds originating in the United Kingdom and Ireland, and identified by The Kennel Club (KC) as having annual registration numbers of 300 puppies or fewer.', ' The need for such a list was first identified in June 2003, with research conducted by the KC to identify the extent of the vulnerability and viability of each breed.', ' It was a joint project, with the KC working with the British and Irish Native Breeds Trust, later to be known simply as the Native Dog Breeds Trust.', ' The breeds on the list have been promoted at events such as Discover Dogs and Crufts, and by asking that owners of these breeds mate their dogs rather than having them spayed.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.670\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adca8215542994ed6169bbc', 'answer': 'John Mark Galecki', 'question': 'Which American actor tries to make his long distance relationship with Priya work in \"The Infestation Hypothesis\"  ', 'supporting_facts': [['The Infestation Hypothesis', 2], ['Johnny Galecki', 0]], 'context': [['Rachel Specter', ['Rachel Sarah Specter (born April 9, 1980) is an American actress and writer, who is best known as the model for the RGX body spray commercials.', ' In addition to her work in commercials, Specter has guest-starred in episodes of \"How I Met Your Mother\", \"Gilmore Girls\", \"What I Like About You\", and \"Entourage\", as well as co-hosted the April 4, 2007 episode of \"Attack of the Show!', '\" and a segment of \"The Feed\" on May 23.', ' In September 2008, Specter began co-starring in the web series \"Long Distance Relationship\" on Crackle.']], ['Endurance running hypothesis', ['The endurance running hypothesis is the hypothesis that the evolution of certain human characteristics can be explained as adaptations to long distance running.', ' The hypothesis suggests that endurance running played an important role for early hominins in obtaining food.', ' Researchers have proposed that endurance running began as an adaptation for scavenging and later for persistence hunting.']], ['The Infestation Hypothesis', ['\"The Infestation Hypothesis\" is the second episode of the fifth season of \"The Big Bang Theory\" that first aired on CBS on September 22, 2011.', ' It is the 89th episode overall.', ' In it, Sheldon (Jim Parsons) becomes worried when Penny acquires a new chair, while Leonard (Johnny Galecki) tries to make his long distance relationship with Priya work.', ' The episode was watched by nearly 12 million viewers in the U.S. and received mixed reviews.']], ['Meredith Kessler', ['Meredith Brooke Kessler (born June 28, 1978) is an American professional triathlete from Columbus, Ohio who races in long distance, non-drafting triathlon events.', ' She took third place at the 2011 ITU Long Distance Triathlon World Championships and has won numerous Ironman and half-Ironman distance races as both an amateur and a professional.', \" She was named USA Triathlon's 2014 Non-Drafting Athlete of the Year.\"]], ['Johnny Galecki', ['John Mark Galecki (born April 30, 1975) is an American actor.', ' He is known for playing David Healy in the ABC sitcom \"Roseanne\" from 1992 to 1997 and Dr. Leonard Hofstadter in the CBS sitcom \"The Big Bang Theory\" since 2007.', ' Galecki also appeared in the films \"National Lampoon\\'s Christmas Vacation\" (1989), \"Prancer\" (1989), \"Suicide Kings\" (1997), \"I Know What You Did Last Summer\" (1997), \"Bookies\" (2003), and \"In Time\" (2011).']], ['Communications in Guam', ['Though Guam is a United States territory, some U.S. long distance plans and courier services list Guam as an international location.', \" As a result of Guam's being added to the North American Numbering Plan (NANP) in 1997, calls made to the U.S., Canada, or other participating countries from Guam (or to Guam from other NANP locations) only require the caller to dial a 1 followed by the area code.\", ' In this way, only domestic charges are incurred between the US and Guam on most carriers.', \" Before Guam's inclusion, calling the U.S. required dialing the international 011 first, thus resulting in higher long distance rates and less frequent calls to the U.S. by relatives in Guam.\", ' Prices of long distance calls to these destinations have dropped significantly to the point where now calling the U.S. from Guam or calling Guam from the U.S. costs the same.']], ['Permanent Roommates', ['Permanent Roommates is an Indian web series created by The Viral Fever(TVF) and Biswapati Sarkar.', ' This series revolves around a young couple,Tanya and Mikesh, who after being in a long distance relationship for 3 years, face the prospect of marriage.', ' Permanent Roommates has been renewed for a third season, which will premiere in 2018.']], ['Made in Chelsea (series 10)', ['The tenth series of Made in Chelsea, a British structured-reality television programme, began airing on 19 October 2015 on E4.', ' The official trailer for the new series was released on 29 September 2015 confirming the start date.', ' It concluded on 4 January 2016 following nine regular episodes, a Christmas special, a New Year special, and an End of Season party hosted by Rick Edwards.', ' This series was the first to include new cast members Emma Walsh, Sam Harney, Tallulah Rufus Isaacs.', ' Richard Dinan also returned to the series having last appeared during the fifth series, and Francis Boulle made a one-off return during the Christmas special.', ' This was also the final series to include original cast member Spencer Matthews, long-running cast member Oliver Proudlock, as well as Millie Wilkinson and Emily Weller, who both made their debuts during the ninth series.', \" The series focused heavily on Sam and Tiff's rocky relationship coming to an end when Tiff admits to cheating on him during the summer and rumours of Sam cheating surface, until the pair eventually reunite.\", \" It also includes Louise and Alik attempting to make their long distance relationship work with obstacles in their way, Binky and JP finally making their relationship official despite commitment issues from his part, and Spencer causing further trouble by hooking up with Ollie's latest love interest Emma.\"]], ['The Heart Machine', ['The Heart Machine is a 2014 romantic thriller film written and directed by Zachary Wigon based on his short film \"Someone Else\\'s Heart\".', \" The film centers on Cody's John Gallagher, Jr. and Virginia's Kate Lyn Sheil long distance relationship that becomes strained when evidence appears to contradict Virginia's background.\", ' The film was released in a limited release on October 24, 2014, by Filmbuff.']], ['Northwestern International University', ['Northwestern International University was one of the first colleges to offer self-directed online programs, which were based on review of prior-earned college credits, professional life-experiences, practical knowledge, research, portfolio work, and the passage of comprehensive examinations *Cite (Northwestern International University Registration Catalog).', ' N.I.U. was a member of the Long Distance Learning Council *Cite (Long Distance Learning Council Catalog).', ' Their admissions process consisted of the initial registration process, student selection, and the review of student work and experience.', ' Students had to show proof of passing content specific exams before being considered for school admission.', ' They were also required to pass comprehensive exams at the completion of their respective program.', ' Furthermore, students were expected to complete a Practicum Learning Portfolio Log.', ' The time-requirement for portfolio hours varied by subject matter.', ' Lastly, students had to successfully complete and present research, before N.I.U. would issue their degree *Cite(Northwestern International University Registration Catalog).']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.672\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8ee6915542990e94052bad', 'answer': 'third', 'question': 'What season was the character introduced that becomes the main antagonist in the  following season, from the animated television series created by Bryan Konietzko and Michael Dante DiMartino as a sequel to \"\", which aired from 2005 to 2008? ', 'supporting_facts': [['Kuvira', 0], ['Kuvira', 1], ['The Legend of Korra', 0], ['The Legend of Korra', 1]], 'context': [['Bolin (The Legend of Korra)', ['Bolin (愽林 , Bó Lín ) is a major fictional character in Nickelodeon\\'s animated television series \"The Legend of Korra\", which aired from 2012 to 2014.', ' The character and the series, a sequel to \"\", were created by Michael Dante DiMartino and Bryan Konietzko.', ' He is voiced by P. J. Byrne.', ' Bolin is able to manipulate the classical element of earth, which is known as earthbending.', ' It is revealed in the third season that he is also able to create and control lava, which is a very rare sub-ability called lavabending.']], ['Zaheer', ['Zaheer is a major recurring character in Nickelodeon\\'s animated television series \"The Legend of Korra\" (a sequel to \"\").', ' While he serves as the main antagonist of \"Book Three: Change\", his actions have lingering effects on Avatar Korra and the series\\' plot in the following book.', ' The character was created by Michael Dante DiMartino and Bryan Konietzko and is voiced by Henry Rollins.']], ['List of Avatar: The Last Airbender episodes', ['\"\" is a 61-episode American animated television series created by Michael Dante DiMartino and Bryan Konietzko.', ' It first aired on February 21, 2005, on Nickelodeon with a one-hour series premiere and concluded its run with a two-hour TV movie on July 19, 2008.', ' The \"Avatar: The Last Airbender\" franchise refers to each season as a \"Book\", in which each episode is referred to as a \"chapter\".', ' Each \"Book\" takes its name from one of the elements that Aang, the protagonist, must master: Water, Earth, and Fire.', \" The show's first two seasons each consisted of 20 episodes, while the third season had 21.\", ' In addition to the three seasons, there were two recap episodes and three \"shorts\".', ' The first recap summarized the first eighteen episodes while the second summarized season two.', ' The first self-parody was released via an online flash game.', ' The second and third were released with the Complete Second Season Box Set DVD.', ' The entire series has been released on DVD in Region One, Region Two and Region Four.']], ['Iroh', ['General Iroh (艾洛 , Aì Luò ) is a fictional character in Nickelodeon\\'s animated television series \"\".', ' Created by Michael Dante DiMartino and Bryan Konietzko, the character was voiced by Mako Iwamatsu in season one and season two and, due to Mako\\'s death, by Greg Baldwin, in season three and the sequel series \"The Legend of Korra\".']], ['Avatar: The Last Airbender (season 2)', ['Season Two (Book Two: Earth) of \"\", an American animated television series on Nickelodeon, first aired its 20\\xa0episodes from March 17, 2006 to December 1, 2006.', ' The season was created and produced by Michael Dante DiMartino and Bryan Konietzko, and starred Zach Tyler Eisen, Mae Whitman, Jack DeSena, Jessie Flower, Dante Basco, Dee Bradley Baker, Mako Iwamatsu and Grey DeLisle as the main character voices.']], ['Avatar: The Last Airbender (season 1)', ['Season one (Book One: Water) of \"\", an American animated television series produced by Nickelodeon Studios, aired 20 episodes from February 21, 2005 to December 2, 2005.', ' The series was created by Michael Dante DiMartino and Bryan Konietzko, and starred Zach Tyler Eisen, Mae Whitman, Jack DeSena, Dante Basco, Dee Bradley Baker, Mako Iwamatsu and Jason Isaacs as the main character voices.']], ['The Legend of Korra', ['The Legend of Korra is an American animated television series that aired on the Nickelodeon television network from 2012 to 2014.', ' It was created by Bryan Konietzko and Michael Dante DiMartino as a sequel to \"\", which aired from 2005 to 2008.', ' Animated in a style strongly influenced by anime, the series is set in a fictional universe in which some people can manipulate, or \"bend\", the elements of water, earth, fire, or air.', ' Only one person, the \"Avatar\", can bend all four elements, and is responsible for maintaining balance in the world.', ' The series follows Avatar Korra, the reincarnation of Aang from the previous series, as she faces political and spiritual unrest in a modernizing world.']], ['Kuvira', ['General Kuvira (古維拉 , Gǔ Wéi Lā ) is a fictional character and a character in \"The Legend of Korra\", created by Michael Dante DiMartino and Bryan Konietzko.', ' Introduced in the third season of the series, she becomes the main antagonist of the fourth season.', ' Kuvira was created with similar characteristics to the portrayal of protagonist Korra in prior seasons to highlight the changes she had made over the series.', \" Kuvira's character has been mostly met with positive reception.\", ' Critics note her motives as being understandable, while her actions are given political analogues.']], ['List of The Legend of Korra episodes', ['\"The Legend of Korra\" is an American animated television series created by Michael Dante DiMartino and Bryan Konietzko.', ' A sequel to \"\", the series first aired on Nickelodeon in 2012.', ' Like its predecessor, the series is set in a fictional world inspired by Asian and Inuit cultures, and inhabited by people who can manipulate the elements of water, earth, fire or air through an ability called \"bending.\"', ' One person, the \"Avatar,\" has the ability to bend all four elements.', \" Reincarnating in turn among the world's four nations, the Avatar is responsible for maintaining peace, harmony, and balance in the world.\", ' Korra, the series\\' protagonist, is the next incarnation of the Avatar after Aang of \"Avatar: The Last Airbender\".', ' Four seasons with a total of 52 episodes have aired.']], ['Avatar: The Last Airbender (season 3)', ['Season Three (Book Three: Fire) of \"\", an American animated television series on Nickelodeon, first aired its 21 episodes from September 21, 2007 to July 19, 2008.', ' The season was created by Michael Dante DiMartino and Bryan Konietzko, and starred Zach Tyler Eisen, Mae Whitman, Jack DeSena, Jessie Flower, Dante Basco, Dee Bradley Baker, Greg Baldwin, Grey DeLisle and Mark Hamill as the main character voices.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.673\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac1c7605542994ab5c67e14', 'answer': 'German', 'question': \"What was the nationality of Bernhard Heiden's teacher?\", 'supporting_facts': [['Bernhard Heiden', 0], ['Paul Hindemith', 0]], 'context': [['Iwan Knorr', ['Iwan Knorr (3 January 1853 – 22 January 1916) was a German composer and teacher of music.', ' A native of Mewe, he attended the Leipzig Conservatory where he studied with Ignaz Moscheles, Ernst Friedrich Richter and Carl Reinecke.', ' In 1874 he became a teacher and in 1878 director of music theory instruction at the Imperial Kharkiv Conservatory, in what is now Ukraine.', ' In 1883 he settled in Frankfurt, where he joined the faculty of the Hoch Conservatory; in 1908 he became director of the school.', ' As a teacher he exerted great influence; among his pupils were Bernhard Sekles, Ernest Bloch, Vladimir Sokalskyi, Ernst Toch, Roger Quilter, Hans Pfitzner, and Cyril Scott.', ' Knorr died in Frankfurt.']], ['Johann Bernhard Logier', ['Johann Bernhard Logier (9 February 1777 – 13 February 1846) was a German composer, teacher, inventor, and publisher resident in Ireland for most of his life.']], ['Eberhard Achterberg', ['Eberhard Achterberg (9 January 1910 in Oliva, West Prussia, now part of Gdańsk, Poland – 11 August 1983 in Neumünster) was a religious scholar, a journalist, a high-ranking Nazi official in the Amt Rosenberg and later a leading member of the German Unitarian Religious Community and school and university teacher.', ' He was the father of the psychologist Bernhard Achterberg.']], ['Bernhard Heiden', ['Bernhard Heiden (b. Frankfurt-am-Main, August 24, 1910; d. Bloomington, IN, April 30, 2000) was a German and American composer and music teacher, who studied under and was heavily influenced by Paul Hindemith.', ' Bernhard Heiden, the son of Ernst Levi and Martha (Heiden-Heimer) was originally named Bernhard Levi, but he later changed his name.']], ['Cornelius Bernhard Hanssen', ['Cornelius Bernhard Hanssen (25 February 1864 – 16 April 1939) was a Norwegian teacher, shipowner and politician for the Liberal Party.']], ['Goudreau Museum of Mathematics in Art and Science', ['The Goudreau Museum of Mathematics in Art and Science was a museum of math that was open from 1980–2006 in Long Island, New York.', ' The museum was named after mathematics teacher Bernhard Goudreau, who had died in 1985, and featured many of the 3-dimensional solid models, oversized wooden math games, and puzzles built by Goudreau and his former students.', ' After the museum closed, Glen Whitney, a former math professor, decided to open the Museum of Mathematics in Manhattan (New York City), which opened in December 2012.']], ['Paul Hindemith', ['Paul Hindemith ( ) (16 November 1895 – 28 December 1963) was a prolific German composer, violist, violinist, teacher and conductor.', ' Notable compositions include his song cycle \"Das Marienleben\" (1923), \"Der Schwanendreher\" for viola and orchestra (1935), and opera \"Mathis der Maler\" (1938).', ' Hindemith\\'s most popular work, both on record and in the concert hall, is likely the \"Symphonic Metamorphosis of Themes by Carl Maria von Weber\", written in 1943.']], ['Hermann Zilcher', ['Hermann Zilcher (born August 18, 1881 in Frankfurt am Main; † 1 January 1948 in Würzburg) was a German composer, pianist, conductor and music teacher.', ' He was the father of actress Eva Zilcher (1920-1994) and the conductor Heinz Reinhart Zilcher (1906-1967).', 'Zilcher received early piano lessons from his father, the composer and piano pedagogue Paul Zilcher (1855-1943), who was known as a composer of didactic piano and chamber music.', ' The son studied from 1897 at the Dr. Hoch Conservatory in Frankfurt, piano with James Kwast, counterpoint and morphology with Iwan Knorr and composition with Bernhard Scholz.', ' At graduation he was awarded the Mozart Prize.', ' In Frankfurt.', ' In 1901 he moved to Berlin, where he quickly established himself mainly as a pianist for singers and instrumentalists, with concert tours, which made him internationally known in the US and in Europe.', ' In 1905 he returned to Frankfurt as a piano teacher at the Dr. Hoch Conservatory.', ' In 1908 he was appointed by Felix Mottl as a piano professor and in 1916 as a composition professor at the Academy of Music in Munich.', ' In Munich, he worked closely with the head of the Munich Kammerspiele, Otto Falckenberg (1873-1947), for whom he wrote incidental music.', ' In 1920 he became director of the Bavarian State Conservatory in Würzburg, and founded in 1922, the Würzburg Mozart Festival, which soon became internationally famous.', ' For these accomplishments Zilcher was appointed in 1924 Privy Councillor by the Bavarian government and the University of Würzburg awarded him an honorary doctorate.']], ['Johann Bernhard Basedow', ['Johann Bernhard Basedow (September 11, 1724, – July 25, 1790) was a German educational reformer, teacher and writer.', ' He founded the Philanthropinum, a short-lived but influential progressive school in Dessau, and was the author of \"\"Elementarwerk\"\", a popular illustrated textbook for children.']], ['Bernhard Kontarsky', ['Bernhard Kontarsky (born 26 April 1937 in Iserlohn) is a German conductor, pianist, and teacher.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.674\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8986fd55429938390d4046', 'answer': 'animation', 'question': 'What technique does Cam Clarke and Akira have in common?', 'supporting_facts': [['Cam Clarke', 0], ['Akira (1988 film)', 0]], 'context': [['John Clarke (mountaineer)', ['John Clarke, CM (February 25, 1945 – January 23, 2003) was a Canadian explorer, mountaineer, conservationist, and wilderness educator.', ' He was born in Ireland to Brigit Ann Clarke (née Conway) and Thomas Kevin Clarke, and died in Vancouver, British Columbia of a brain tumor.', ' From 1964 until his death in 2003 Clarke spent at least six months of each year on extended backcountry trips, usually into the Coast Mountains of British Columbia using the technique of dropping food caches from small planes along an intended route, then traveling that route for weeks at a time.', ' His routes regularly led him along the high ridges and glaciated icefields of the west coast, and allowed him to make hundreds of first ascents of the many mountains along the way.', ' Many of these trips exceeded 30 days in length, and were often done solo, simply because nobody could afford the time to accompany him.']], ['Akira (given name)', ['Akira (あきら, アキラ ) is a common Japanese given name.', ' There are several kanji for Akira.', ' A popular kanji is 明 which means \"bright\", \"intelligent\", or \"clear\".', ' Though Akira is normally used to name males, it can be a female name as well.']], ['Cam Clarke', ['Cameron Arthur \"Cam\" Clarke (born November 6, 1957) is an American voice actor and singer, known for his voice-work in animation and video games.', ' He is best known for providing the voices of Leonardo and Rocksteady in the original \"Teenage Mutant Ninja Turtles\" animated series and Shotaro Kaneda in the 1989 original English dub of \"Akira\".', ' He often voices teenagers and other similarly young characters.', ' One of his prominent roles in video games was voicing Liquid Snake in the \"Metal Gear\" series.']], ['Common area maintenance charges', ['Common Area Maintenance charges, or CAM for short, are one of the net charges billed to tenants in a commercial triple net (NNN) lease, and are paid by tenants to the landlord of a commercial property.', ' A CAM charge is an additional rent, charged on top of base rent, and is mainly composed of maintenance fees for work performed on the common area of a property.']], ['Shaky camera', ['Handheld camera, shaky cam, queasy cam, queasicam, hand-held camera or free camera is a cinematographic technique where stable-image techniques are purposely dispensed with.', ' The camera is held in the hand, or given the appearance of being hand-held, and in many cases shots are limited to what one photographer could have accomplished with one camera.', ' Shaky cam is often employed to give a film sequence an ad hoc, electronic news-gathering, or documentary film feel.', ' It suggests unprepared, unrehearsed filming of reality, and can provide a sense of dynamics, immersion, instability or nervousness.', ' The technique can be used to give a pseudo-documentary or \"cinéma vérité\" appearance to a film.']], ['Transfer (propaganda)', ['Transfer is a technique used in propaganda and advertising.', ' Also known as association, this is a technique of projecting positive or negative qualities (praise or blame) of a person, entity, object, or value (an individual, group, organization, nation, patriotism, etc.) to another in order to make the second more acceptable or to discredit it.', ' It evokes an emotional response, which stimulates the target to identify with recognized authorities.', ' Often highly visual, this technique often utilizes symbols superimposed over other visual images.', \" An example of common use of this technique in the United States is for the President to be filmed or photographed in front of the country's flag.\", ' Another technique used is celebrity endorsement.']], ['Computer-aided manufacturing', ['Computer-aided manufacturing (CAM) is the use of software to control machine tools and related ones in the manufacturing of workpieces.', ' This is not the only definition for CAM, but it is the most common; CAM may also refer to the use of a computer to assist in all operations of a manufacturing plant, including planning, management, transportation and storage.', ' Its primary purpose is to create a faster production process and components and tooling with more precise dimensions and material consistency, which in some cases, uses only the required amount of raw material (thus minimizing waste), while simultaneously reducing energy consumption.']], ['Mosaic (film)', ['Mosaic is an animated superhero film about a new character created by Stan Lee.', ' It features the voice of Anna Paquin as Maggie Nelson with supporting roles done by Kirby Morrow, Cam Clarke, Garry Chalk, Ron Halder, and Nicole Oliver.', ' It was released under the \"Stan Lee Presents\" banner, which is a series of direct-to-DVD animated films distributed by POW Entertainment with Anchor Bay Entertainment.', ' The story was by Stan Lee, with the script by former X-Men writer Scott Lobdell.']], ['Akira (1988 film)', [\"Akira is a 1988 Japanese adult animated science fiction film directed by Katsuhiro Otomo, produced by Ryōhei Suzuki and Shunzō Katō, and written by Otomo and Izo Hashimoto, based on Otomo's manga of the same name.\"]], ['Jesus Green', ['Jesus Green is a park in the north of central Cambridge, England.', ' It is located north of Jesus College, hence the name.', ' Jesus Ditch runs along the southern edge Jesus Green.', ' On the northern edge of Jesus Green is the River Cam, with Chesterton Road (the A1303) on the opposite side.', ' To the east is Victoria Avenue and beyond that Midsummer Common, common land that is still used for grazing.', ' Victoria Avenue crosses the Cam at Victoria Bridge, connecting to Chesterton Road, at the northeastern corner of Jesus Green.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.675\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7d109855429909bec7692f', 'answer': '1978', 'question': 'The director of Panic 5 Bravo was born in what year?', 'supporting_facts': [['Panic 5 Bravo', 0], ['Kuno Becker', 0]], 'context': [['Paul Bravo', ['Paul Bravo (born June 19, 1968 in Campbell, California) is a former American soccer midfielder and forward who played six seasons in Major League Soccer, two in the American Professional Soccer League and two in the USISL.', \" He also earned four caps, scoring one goal, with the United States men's national soccer team.\", ' After his retirement from playing, Bravo served for several years as an assistant coach in both Major League Soccer and the NCAA and was most recently Technical Director for the Colorado Rapids.']], ['Kuno Becker', ['Eduardo Kuno Becker Paz (born January 14, 1978) is a Mexican actor who has worked in telenovelas, Mexican cinema and U.S. cinema, but is best known for his portrayal of Ruben Berrizabal in \"Soñadoras\" and Santiago Muñez in the football movie \"Goal!', '\" and following sequels.']], ['Jake Sinclair (musician)', ['Jake Sinclair (born March 7, 1985) is an American record producer, audio engineer, mixing engineer, multi-instrumentalist, vocalist, and songwriter.', ' His production, engineering, songwriting, and mixing credits include Weezer, Fall Out Boy, Panic!', ' at the Disco, 5 Seconds of Summer, Pink, New Politics, Andrew McMahon in the Wilderness, Gin Wigmore, and Train.', ' Sinclair co-wrote and produced Panic!', ' at the Disco\\'s \"Death of a Bachelor\" album (which debuted at number one on the US Billboard 200) and produced Weezer\\'s 2016 \"Weezer (White Album)\".', ' Both were nominated for Best Rock Album at the 59th Annual Grammy Awards.', ' He co-wrote and produced \"Uma Thurman\" by Fall Out Boy, which debuted at number one on the U.S. iTunes Chart, reached number 22 on the Billboard Hot 100, and was certified 2X Platinum by the RIAA in December 2015.', ' Sinclair received a Grammy nomination for Album of the Year for his work as engineer and bassist on Taylor Swift\\'s \"Everything Has Changed\" alongside producer Butch Walker.', ' He co-wrote and produced the debut single, \"She Looks So Perfect\" by 5 Seconds of Summer that peaked at number one in over five countries and won \"Song of the Year\" at the 2014 ARIA Awards.', ' Sinclair is the former bassist of the indie/pop rock band The Films and the lead singer and producer of the indie pop duo Alohaha.']], ['C. E. Gatchalian', ['C.E. \"Chris\" Gatchalian (born June 5, 1974) is a Canadian playwright, born in Vancouver, British Columbia to Filipino parents, he holds an MFA in Creative Writing and Theatre from the University of British Columbia.', ' His play \"Motifs & Repetitions\" aired on Bravo!', ' (Canada) in 1997 and on the Knowledge in 1998.', ' His other produced plays include \"Claire\", \"Crossing\", \"Broken\" and \"People Like Vince\", a play for young audiences about mental health.', ' His latest play, \"Falling in Time\", had its world premiere in Vancouver in November 2011 and was published by Scirocco Drama in 2012.', \" In 2013 he won the Dayne Ogilvie Prize, a prize presented by the Writers' Trust of Canada to an openly LGBT writer.\"]], ['Rumen Petkov', ['Rumen Petkov (Bulgarian: Румен Петков ) (born 26 January 1948) is a Bulgarian animator and comic creator.', ' His influence spawned a new generation of young Bulgarian comic book artists as Vladimir Nedialkov, Koko Sarkisian, Ivan Kirjakov and others.', ' He was one of the main artists of the comics magazine DUGA (Rainbow), which was the most popular comics for several generations of Bulgarian children.', ' His most popular cartoon is \"The Adventures of Choko the Stork and Boko the Frog\" which was popular in Bulgaria during the 1970s and 1980s.', ' Other famous animated films he directed are \"Friends of Gosho the Elephant\", \"Treasure Planet\", etc.', \" He has won the Grand Prize at the Ottawa Animation Festival and the Palme d'Or at the Cannes Film Festival.\", ' Recently Rumen Petkov has worked as a writer, storyboard artist, animation director and director on some episodes of \"Johnny Bravo\", \"Dexter\\'s Laboratory\", \"Cow and Chicken\", \"I Am Weasel\", \"The New Woody Woodpecker Show\" and other series.', ' He has said about animation: \"Animation will never die because it\\'s like music, because it\\'s like running with the wind, because it\\'s funny.\"']], ['Stéphane Aubier', ['Stéphane Aubier (born October 8, 1964) is a Belgian film director and screenwriter.', ' In 2009, he wrote and directed the animated film \"A Town Called Panic\" along with Vincent Patar.', ' It premiered at the 2009 Cannes Film Festival and was the first stop-motion film to be screened at the festival.', ' In 2013, he co-directed with Patar and Benjamin Renner the film \"Ernest & Celestine\", which received widespread critical acclaim.', ' The film received three Magritte Awards, including Best Film and Best Director for Aubier and Patar.', ' It also received a nomination at the 86th Academy Awards, in the category of Best Animated Feature.']], ['Panic 5 Bravo', ['Panic 5 Bravo is an action-thriller film directed by Kuno Becker about American paramedics that become trapped on the Mexican side of the border and terrorized by a violent psychopath.', ' It was released in the U.S. by Pantelion Films.']], ['The Mins', ['The Mins are a Georgian Alternative / New Progressive Rock band established in 2011 by Zviad Mgebrishvili.', ' The band played its first live gig in 2011 on Altervision Newcomers.', ' After that the band started to work hard on their repertoire.', ' They mostly performs original songs and only rarely covers.', ' The main songwriter in the band is Zviad Mgebrishili.', ' Some songs are written by Shota Gvinepadze (keyboard) as well.', ' The band has four music videos on the following songs: \"Blind World\", \"O.W.L.\", \"My Lover is a Killer\" and \"I Don\\'t Give a Foot\".', ' Zviad Mgebrishvili was participating in the TV show \"Akhali Khma\" [\"The Voice of Georgia\"] in 2013 very successfully (5 stages).', ' The band has performed on many festivals and concerts.', ' The band had their first big solo concert in Tbilisi Eventhall 26 May 2014, where they had presentation of their first EP, named \"Blind World\" (released same year, included 5 songs).', ' The band has an honor to be warm up of \"Faithless\" (Tbilisi Summer Set 2014) and \"Archive\" (Tbilisi Open Air/Altervision 2015, where apart from Archive - Placebo, Beth Hart and Black Label Society were the headliners).', ' Zaza Mgebrishvili has left the band in 2015 and new bass player and backing vocal of the band is Nika Abesadze who used to play with Zviad Mgebrishvili early years in the university rock band \"Sunny Universe\".', ' The band is now recording their first album \"First Minute\" in the Bravo Records sound recording studio that will be released in the Winter of 2015.']], ['Vincent Patar', ['Vincent Patar (born 2 September 1965) is a Belgian film director and screenwriter.', ' In 2009 he wrote and directed the animated film \"A Town Called Panic\" along with Stéphane Aubier.', ' It premiered at the 2009 Cannes Film Festival and was the first stop-motion film to be screened at the festival.', ' In 2013 he co-directed with Aubier and Benjamin Renner the film \"Ernest & Celestine\", which received widespread critical acclaim.', ' The film received three Magritte Awards, including Best Film and Best Director for Patar and Aubier.', ' It also received a nomination at the 86th Academy Awards, in the category of Best Animated Feature, to be held on 2 March 2014.']], ['Ann Lewis (musician)', ['Ann Lewis (アン・ルイス , An Ruisu , born 5 June 1956 in Takarazuka, Hyōgo, Japan) is a Japanese singer, popular in Japan in the 1970s and 80s.', ' She was born to an American father and a Japanese mother.', ' She has one brother and a son, Myuji, who is also a singer in Japan.', ' She was married to Masahiro Kuwana, another Japanese singer, from 1980 to 1984.', ' Her many hits include the popular song \"Roppongi Shinju\", \"Good Bye My Love\" and many others which have been covered by other Asian artists.', ' She semi-retired from show-business in the 1990s, suffering from chronic panic attacks, and settled down in Los Angeles.', ' She released a few self-covers albums in the 2000s.', ' She has been active as a Creative Director, Consultant and Designer.', ' Works include Interior designs, (private homes to business offices, restaurants and shops), releasing a line of original jewelry, Creating original Animation, Logos and other projects.', ' She has also been involved as the President, COO and marketing consultant for several software companies in the USA.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.676\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a82edae55429966c78a6a9f', 'answer': '1986', 'question': 'Swiss music duo Double released their best known single \"The Captain of Her Heart\" in what year?', 'supporting_facts': [['Blue (Double album)', 1], ['Double (band)', 0]], 'context': [['Feargal Sharkey (album)', ['Feargal Sharkey is the first solo album of former Undertones singer Feargal Sharkey.', ' The album was released in 1985, peaking at #12 in the UK and contains Sharkey\\'s best known single \"A Good Heart\" his only No. 1.']], ['Devils Ball', ['\"Devils Ball\" is a song by Swiss duo Double, released as the lead single from their second studio album \"Dou3le\".', ' The single was released in 1987, and featured a guest appearance from Herb Alpert, who played trumpet on the track.']], ['The Captain of Her Heart', ['\"The Captain of Her Heart\" is a single by the Swiss duo Double in 1985.', ' Taken from their 1985 album \"Blue\", the song is a ballad about a girl who stops waiting for her absent lover to return.', ' The song was an international success, reaching No. 8 in the UK Singles Chart and No. 16 on the \"Billboard\" Hot 100.', ' The song also made Double the first Swiss act to hit the Top 40 in the Billboard Hot 100.']], ['Double (band)', ['Double (pronounced \"doo-blay\") was a Swiss music duo best known for their hit single \"The Captain of Her Heart\".']], ['Blue (Double album)', ['Blue is the first full-length album from Swiss band Double.', ' In addition to containing updated versions of two of the band\\'s earlier singles (\"Woman of the World\" and \"Rangoon Moon\"), the album included the international smash hit, \"The Captain of Her Heart\", a plaintive, atmospheric, piano-led ballad which was an immediate success throughout Europe upon its 1986 single release.', ' Follow-up singles \"Your Prayer Takes Me Off\" and \"Tomorrow\" were less successful.']], ['Kurt Maloo', ['Kurt Maloo (born Kurt Meier, April 16, 1953 in Zurich, Switzerland) is a Swiss singer-songwriter, composer, and record producer.', ' He first achieved international success in 1986, as the singer and front man of the duo Double with the hit single, \"The Captain of Her Heart\".']], ['Parno Graszt', ['Parno Graszt is a Roma (i.e. \"Gypsy\") music ensemble from Paszab, Hungary founded in 1987.', ' \"Parno Grast\" means \"white horse\" in the Romany language, with \"graszt\" using the Hungarian orthography \\'sz\\' for \\'s\\'.', ' In the Roma culture white is symbol of purity and horse is a symbol of freedom.', ' Their debut album \"Hit the piano\" reached Number 7 on the World Music Chart Europe in October 2002.', ' Hungarian Television and the BBC produced in 2004 a music documentary about Parno Graszt.', ' After their second album, \"Járom az utam\" (2004), Parno Graszt was voted in the top 10 for \"best artist of year\", 2005, by the Swiss music magazine \"Vibrations\".', ' In 2016, they competed in A Dal, the national final selection for Hungary in the Eurovision Song Contest with the song \"Már nem szédülök\", and reached the final.']], ['Move It Like This (song)', ['\"Move It Like This\" is a song recorded by the Bahamian pop group Baha Men.', ' It was released in February 2002 as the second single from the album, \"Move It Like This\".', ' The song reached number 13 on the New Zealand RIANZ list, number 13 on the Canadian Singles Chart and number 65 on the Swiss Music Charts.', ' The song was also featured on the 2002 compilation album \"Now That\\'s What I Call Music!', ' 10\".']], ['Stick Figure Neighbourhood', ['Stick Figure Neighbourhood was the first album by the Burlington band Spoons.', ' Released in 1981, it received some airplay on college stations, particularly the songs \"Conventional Beliefs\" and \"Red Light\".', ' It was their next album, \"Arias & Symphonies\", and its best known single, \"Nova Heart\", that were to launch them to fame.']], ['2005 in Swiss music', ['2005 was a big year for Swiss music, with the charts becoming steadier yet less predictable than they had been in previous years.', ' The year saw many chart debuts from both Swiss and international acts, and saw two novelty songs share a combined total of over ten weeks at the singles chart number one spot.', ' Internationally, the Swiss also saw Vanilla Ninja take the country to their best Eurovision Song Contest position in twelve years.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.676\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a891e1e5542993b751ca913', 'answer': 'Nicholas Farrar Hughes', 'question': 'Who was a grandson of Aurelia Plath and was a fisheries biologist?', 'supporting_facts': [['Aurelia Plath', 0], ['Nicholas Hughes', 0]], 'context': [['NOAAS Reuben Lasker (R 228)', ['NOAAS \"Reuben Lasker\" is a National Oceanic and Atmospheric Administration (NOAA) fishery research vessel.', \" The ship's namesake, Reuben Lasker, was a fisheries biologist who served with the Southwest Fisheries Center, National Marine Fisheries Service, and taught at the Scripps Institution of Oceanography.\"]], ['Taningia', ['Taningia is a genus of squid in the family Octopoteuthidae.', ' It has one confirmed species, \"Taningia danae\".', ' Another species, \"Taningia persica\", is suggested, but has been questioned.', ' The genus is named after Danish fisheries biologist Aage Vedel Taaning (1890-1958).']], ['Robert J. Behnke', ['Dr. Robert J. Behnke (December 30, 1929 – September 13, 2013) was an American fisheries biologist and conservationist who was recognized as a world authority on the classification of salmonid fishes.', ' He was popularly known as \"Dr. Trout\" or \"The Trout Doctor\".', ' His seminal work, \"Trout and Salmon of North America\", was published in 2002.', ' He wrote a regular column for \"Trout Magazine\", the quarterly publication of Trout Unlimited.', ' He was a fisheries biologist with the U.S. Fish and Wildlife Service in the Colorado Cooperative Fish and Wildlife Research Unit and a professor at Colorado State University in the 1970s.', ' He became a Professor Emeritus at the Department of Fishery and Wildlife Biology at Colorado State University.']], ['Archibald Gowanlock Huntsman', ['Archibald Gowanlock Huntsman (November 23, 1883 – August 8, 1973) was a Canadian academic, oceanographer, and fisheries biologist.', ' He is best known for his research on Atlantic salmon and inventing the fast freezing of fish fillets in 1929.']], ['Kamakichi Kishinouye', ['Kamakichi Kishinouye (岸上 鎌吉 , Kishinoue Kamakichi , 29 November 1867 – 22 November 1929) was a Japanese fisheries biologist and cnidariologist and a professor of the Imperial University of Tokyo (Faculty of Agriculture) between 1908 and 1928.', ' Kishinouye died in Chengtu of a sudden illness while on a collecting expedition to China.']], ['Nicholas Hughes', ['Nicholas Farrar Hughes (January 17, 1962 – March 16, 2009) was a fisheries biologist known as an expert in stream salmonid ecology.', ' Hughes was the son of the American poet Sylvia Plath and English poet Ted Hughes and the younger brother of artist and poet Frieda Hughes.', ' He and his sister were well known to the public through the media when he was a small child, especially after the well-publicized suicide of his mother.', ' Hughes held dual British/American citizenship.']], ['Otto Plath', ['Otto Emil Plath (April 13, 1885\\xa0– November 5, 1940) was a German American author, a professor of biology and German at Boston University, and an entomologist, with a specific expertise on bees.', ' He was the father of American poet Sylvia Plath, Warren Plath, and the husband of Aurelia Plath.', ' He wrote the 1934 book, \"Bumblebees and Their Ways\".', ' He is notable for being the probable subject of one of his daughter\\'s most well-known poems, \"Daddy\".']], ['C.G. Johannes Petersen', ['Carl Georg Johannes Petersen (24 October 1860 – 11 May 1928) was a Danish marine biologist, especially fisheries biologist.', ' He was the first to describe communities of benthic marine invertebrates and is often considered a founder of modern fisheries research.', ' Especially he was the first to use the Mark and recapture method which he used to estimate the size of a Plaice population.', ' The Lincoln-Petersen method also known as the Petersen-Lincoln index is named after him.']], ['David Cushing', ['David Henry Cushing FRS was an English born fisheries biologist, who is credited with the development the match/mismatch hypothesis as an explanation for reduced fish stocks as associated with climatic variability.', ' As opposed to other important fisheries biologists, such as Daniel Pauly and Carl J. Walters, Cushing was a proponent of keeping fisheries open to the point of collapse.']], ['Aurelia Plath', ['Aurelia Frances Plath (née Schober; April 26, 1906 – March 11, 1994) was the wife of Otto Emil Plath, the mother of the American poet Sylvia Plath, and her brother Warren, and the grandmother of Frieda Rebecca Hughes and Nicholas Farrar Hughes.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.676\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae1b8075542997283cd2262', 'answer': 'Marvin John Heemeyer', 'question': \"What muffler repair shop owner inspired Andrey Zvyagintsev's film Leviathan?\", 'supporting_facts': [['Leviathan (2014 film)', 1], ['Marvin Heemeyer', 0]], 'context': [['Loveless (film)', ['Loveless (Russian: Нелюбовь ) is a 2017 Russian drama film directed by Andrey Zvyagintsev.', ' The story concerns two separated parents living apart whose affections are long forgotten and whose relationship has become loveless.', ' They are temporarily brought together after their only young child becomes a missing person and they attempt to find him.', ' It was shot in Moscow, with international support after the Russian government disapproved of Zvyagintsev\\'s 2014 film \"Leviathan\".', ' \"Loveless\" opened to critical acclaim and it won the Jury Prize at the 2017 Cannes Film Festival.', ' It was selected as the Russian entry for the Best Foreign Language Film at the 90th Academy Awards.']], ['Minneapolis Fire Department Repair Shop', ['The Minneapolis Fire Department Repair Shop is a building in Minneapolis, Minnesota listed on the National Register of Historic Places in 2005.', ' The repair shop was established by the city of Minneapolis to reorganize and consolidate the services of the fire department.', ' The shop was also used to convert horse-drawn fire equipment to motorized vehicles.']], ['Adrien (2015 film)', ['Adrien (French: \"Le Garagiste\" ) is a 2015 Canadian drama film, written and directed by Renée Beaulieu.', \" The film stars Normand D'Amour as Adrien, a small-town automobile repair shop owner who is confronting his mortality as he awaits a kidney transplant.\"]], ['Louden Monorail System in the Auto Repair Shop', [\"The Louden Monorail System in the Auto Repair Shop, also known as McGuire Motor Company and Crandall's Electric Service, is a historic structure located in Fairfield, Iowa, United States.\", ' The monorail system is located in a former auto repair shop along an alley between East Broadway Avenue and East Briggs Avenue.', ' It is the rear, single-story, portion of the building at 117 E. Broadway Ave where the system is located.', ' The storefront portion of the building, also historically associated with the automobile industry, is a two-story brick building built on a stone foundation.', ' The east side of the central business district in Fairfield had become the center for automobile related businesses by the 1920s.', ' Harley Carter bought this building in 1920, and had the monorail system, manufactured by the Louden Machinery Company, installed about 1922.', ' The overhead material handling system is permanently attached to the east wall of the shop.', ' It allowed the mechanics to more easily move the heavy engines and other parts to and from vehicles.', ' The structure was listed on the National Register of Historic Places in 1999.']], ['Robert Giardinelli', ['Robert Giardinelli (1914 in Catania, Italy – 1996 in New York City, New York) was a noted musical instrument craftsman who operated a musical instrument repair shop in New York City.', ' After immigrating to the United States, Giardinelli served in the United States Army during World War II.', ' Starting in the Bronx in 1946, he later moved his music shop to midtown Manhattan, where he remained in business for over 40 years until his retirement.', \" Giardinelli's business included musical mouthpiece manufacturing, a discount retail music store, and a custom repair shop for brass and wind instruments.\", \" Giardinelli's music shop was located on the upper floors at 151 West 46th Street.\", \" His business became a world-renowned stop for musicians during the 1980's.\"]], ['Mikhail Krichman', ['Mikhail Krichman (Михаил Владимирович Кричман; born 1967) is a Russian cinematographer who received a Golden Osella award at the 67th Venice Film Festival for \"Silent Souls\".', ' He photographed all of Andrey Zvyagintsev\\'s films, including \"The Return\" (2003), \"The Banishment\" (2007), \"Elena\" (2010) and \"Leviathan\" (2014).', ' Zvyagintsev claims that Krichman (an engineer by profession) learned his craft by reading \"American Cinematographer\".', ' Krichman also shot \"Miss Julie\" for Norwegian director Liv Ullmann.']], ['Marvin Heemeyer', ['On June 4, 2004, automobile muffler repair shop owner Marvin John Heemeyer drove his armored bulldozer through Granby, Colorado, damaging 13 buildings, with the cost of the damage rounding to an estimated $7 million.', \" Heeymeyer's bulldozer rampage, which targeted other parties of a zoning dispute, ended ignominiously when Heemeyer committed suicide with a handgun inside his Komatsu D355A bulldozer.\", ' Heemeyer added improvised composite armor to his bulldozer consisting of layers of concrete and steel, creating what the media called a \"killdozer\".']], ['Andrey Zvyagintsev', ['Andrey Petrovich Zvyagintsev (Russian: Андре́й Петро́вич Звя́гинцев ; born 6 February 1964) is a Russian film director and screenwriter.', ' He is mostly known for his 2003 film \"The Return\", which won him a Golden Lion at the Venice Film Festival.', ' Following \"The Return\", Zvyagintsev directed \"The Banishment\" and \"Elena\".', ' His film \"Leviathan\" was nominated for the Academy Award for Best Foreign Film in 2014.', ' His most recent film \"Loveless\" won the Jury Prize at the 2017 Cannes Film Festival.']], ['Comparison of Auto Repair Shop Management Systems', ['Auto repair shop management system is an automated system for auto repair workflow management.', ' This is a comparison of auto repair shop management systems which are the most popular so far.']], ['Leviathan (2014 film)', ['Leviathan (Russian: Левиафан , \"Leviafan\") is a 2014 Russian drama film directed by Andrey Zvyagintsev, co-written by Zvyagintsev and Oleg Negin, and starring Aleksei Serebryakov, Elena Lyadova, and Vladimir Vdovichenkov.', \" According to Zvyagintsev, the story of Marvin Heemeyer in the United States inspired him and it was adapted into a Russian setting, but critics compare the story to the more similar biblical story of Naboth's Vineyard, where a King vies for his subjects' land and is motivated by his Queen to obtain it in a sly manner.\", ' The character development of the protagonist parallels another biblical figure, Job.', ' The producer Alexander Rodnyansky has said: \"It deals with some of the most important social issues of contemporary Russia while never becoming an artist\\'s sermon or a public statement; it is a story of love and tragedy experienced by ordinary people\".', ' Critics noted the film as being formidable, dealing with quirks of fate, power and money.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.677\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7759fc5542993569682d60', 'answer': 'Canary Islands, Spain', 'question': 'Where are Teide National Park and Garajonay National Park located?', 'supporting_facts': [['Teide National Park', 0], ['Garajonay National Park', 0]], 'context': [['Hermigua', ['Hermigua is a town and a municipality in the northeastern part of La Gomera in the province of Santa Cruz de Tenerife of the Canary Islands, Spain.', \" It is located 12\\xa0km northwest of the island's capital, San Sebastián de la Gomera.\", ' The Garajonay National Park covers the southern part of the municipality.']], ['Garajonay National Park', ['Garajonay National Park (Spanish: \"Parque nacional de Garajonay\" , ] ) is located in the center and north of the island of La Gomera, one of the Canary Islands (Spain).', ' It was declared a national park in 1981 and a World Heritage Site by UNESCO in 1986.', ' It occupies 40 km (15 sq mi) and it extends into each of the six municipalities on the island.']], ['Teide National Park', ['Teide National Park (Spanish: \"Parque nacional del Teide\" , ] ) is a national park located in Tenerife (Canary Islands, Spain).']], ['Crater Lake National Park', ['Crater Lake National Park is a United States National Park located in southern Oregon.', ' Established in 1902, Crater Lake National Park is the fifth-oldest national park in the U.S. and the only national park in Oregon.', ' The park encompasses the caldera of Crater Lake, a remnant of a destroyed volcano, Mount Mazama, and the surrounding hills and forests.']], ['Glacier National Park Fund', ['The Glacier National Park Fund (established in 1999), is an organization whose main goal is to raise money to support the demands of the Glacier National Park, located in West Glacier, Montana.', ' According to the National Park Service, the Glacier National Park Fund supports park projects, programs, and services in four areas: to preserve the park experience, to provide curriculum-based instruction, to research the park ecosystem, and to prepare for the 2010 Centennial year of the park.', ' The Glacier National Park Fund provided funds for different studies.', ' In July 2007, The Glacier National Park Fund funded a study involving human and bear aversion techniques.', \" Also, in 2009, The Glacier National Park Fund granted $10,000 to enable biologists to learn more about the Bighorn Sheep that inhabit Glacier National Park, along the park's boundary with the Blackfeet Indian Reservation.\", ' The Glacier National Park Fund has an official mascot named Billy Bowman.']], ['Roque Cinchado', ['The Roque Cinchado is a rock formation, regarded as emblematic of the island of Tenerife (Canary Islands, Spain).', ' It lies within the Teide National Park (a World Heritage Site) in the municipality of La Orotava, near the volcano of the same name, in the heart of the island.', ' The Roque Cinchado is one of the largest in the world by altitude, for the entire park totals more than 2000 metres.']], ['Adenocarpus viscosus', ['Adenocarpus viscosus is a shrubby species of flowering plant in the legume family Fabaceae, subfamily Faboideae.', ' It is endemic to the Canary Islands where it is known locally as Codeso del Pico.', ' It can be found above 1800 m on two of the islands, La Palma in Caldera de Tabouriente and Tenerife where it is a dominant shrub in Teide National Park and occurs in parts of Corona Forestal Nature Park and Reserva Especial de las Palomas.']], ['Training centre for release of the Atma-energy', ['Training centre for release of the Atma-energy (German: \"Trainingszentrum zur Freisetzung der Atmaenergie\"), also known as \"Atman Foundation\", was a new religious movement active mainly on the island of Tenerife and in Germany.', ' This sect was originally a splinter group of the Brahma Kumaris and is known for a police and media scare in which an alleged attempt to commit ritual suicide took place in Teide National Park in Tenerife.', ' The group believed in the end of the world but according to the religious studies scholar Georg Schmid and the sociologist Massimo Introvigne had no intention of collective suicide.']], ['Alpine National Park', ['The Alpine National Park is a national park located in the Central Highlands and Alpine regions of Victoria, Australia.', ' The 646000 ha national park is located northeast of Melbourne.', \" It is the largest National Park in Victoria, and covers much of the higher areas of the Great Dividing Range in Victoria, including Victoria's highest point, Mount Bogong at 1986 m and the associated subalpine woodland and grassland of the Bogong High Plains.\", \" The park's north-eastern boundary is along the border with New South Wales, where it abuts the Kosciuszko National Park.\", ' On 7 November 2008 the Alpine National Park was added to the Australian National Heritage List as one of eleven areas constituting the Australian Alps National Parks and Reserves.']], ['North Cascades National Park', ['North Cascades National Park is a U.S. National Park located in the state of Washington.', ' The park is the largest of the three National Park Service units that comprise the North Cascades National Park Service Complex.', ' Several national wilderness areas and British Columbia parkland adjoin the National Park.', ' The park features rugged mountain peaks and protects portions of the North Cascades range.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.678\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8bb7f95542996e8ac889ff', 'answer': 'Korea', 'question': 'Where was the movie filmed starring Choi Yoon-young in 2012?', 'supporting_facts': [['Choi Yoon-young', 2], ['As One (film)', 1]], 'context': [['Revolution (2017 TV series)', ['Revolution () is an upcoming South Korean television series starring Choi Si-won, Kang So-ra and Gong Myung.', \" The series marks Choi Si-won's first acting project after his military service.\", ' It is set to air on tvN starting October 14, 2017 at 21:00 KST, replacing \"Live Up to Your Name, Dr. Heo\".']], ['As One (film)', ['As One (; lit.', ' \"Korea\") is a 2012 South Korean sports drama film starring Ha Ji-won and Bae Doona.', ' It is a cinematic retelling of the first ever post-war Unified Korea sports team which won the gold at the 1991 World Table Tennis Championships in Chiba, Japan.', ' Director Moon Hyun-sung used the foundation of true events to tell the story of a team that united a divided nation for the first time in its painful history.']], ['Passionate Love', ['Passionate Love () is a 2013 South Korean weekend television drama series starring Sung Hoon and Choi Yoon-young.', ' It aired on SBS from September 28, 2013 to March 23, 2014 on Saturdays and Sundays at 20:45 for 47 episodes.']], ['Now and Forever (2006 film)', ['Now and Forever () is a 2006 South Korean film directed by Kim Seong-joong and starring Choi Ji-woo, Jo Han-sun, Choi Sung-kook and Seo Young-hee.', 'it also has a Japanese manga named RENRI NO EDA (連理の枝) Intertwined Branches .']], ['The Terrorist (1995 film)', ['The Terrorist () is a 1995 South Korean film directed by Kim Young-bin, starring Choi Min-soo as the younger brother of a police officer who becomes involved with gangsters.', ' It became a box office hit and earned Choi Min-soo an award for best actor.']], ['Pride and Prejudice (2014 TV series)', ['Pride and Prejudice (Hangul:\\xa0오만과 편견 ; Hanja:\\xa0傲慢과 偏見 ; RR:\\xa0\"Omangwa Pyeongyeon \" ) is a 2014 South Korean television series starring Choi Jin-hyuk, Baek Jin-hee, Choi Min-soo, Lee Tae-hwan and Son Chang-min.', ' It aired on MBC from October 27, 2014 to January 13, 2015 on Mondays and Tuesdays at 22:00 for 21 episodes.']], ['Emperor of the Sea', ['Emperor of the Sea (; literally \"Sea God\") is a South Korean television drama series starring Choi Soo-jong, Chae Shi-ra, Song Il-gook، Soo Ae and Chae Jung-an It aired on KBS2 from November 24, 2004 to May 25, 2005 on Wednesdays and Thursdays at 21:55 for 51 episodes.', ' The period drama is based on Choi In-ho\\'s 2003 novel \"Hae-sin\", which depicts the life of Jang Bogo, who rises from a lowly slave to a powerful maritime figure who dominated the East Asia seas and international trade during the Unified Silla Dynasty.']], ['My Dear Cat', ['My Dear Cat () is a 2014 South Korean daily drama starring Choi Yoon-young, Hyun Woo, Choi Min, and Jun Hyoseong.', ' It aired on KBS1 from June 9 to November 21, 2014 on Mondays to Fridays at 20:25 for 119 episodes.']], ['Twenty Again', ['Twenty Again (; lit.', ' \"Twenty Years Old for the Second Time\") is a 2015 South Korean television series starring Choi Ji-woo, Lee Sang-yoon, Choi Won-young, Kim Min-jae, and Son Na-eun.', ' It aired on tvN from August 28 to October 17, 2015 on Fridays and Saturdays at 20:30 for 16 episodes.']], ['Choi Yoon-young', ['Choi Yoon-young (born September 25, 1986) is a South Korean actress.', ' After passing the 21st KBS actors\\' auditions in 2008, Choi began playing supporting roles in the network\\'s dramas, notably in \"King of Baking, Kim Takgu\" (2010) and \"My Daughter Seo-young\" (2012).', ' She then appeared twice on the big screen in 2012: in the short film \"Endless Flight\" in omnibus \"Horror Stories\", and the table tennis sports film \"As One\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.679\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a85ab905542994c784ddb35', 'answer': 'The Saturdays', 'question': 'Two positions were filled in The Voice of Ireland b which British-Irish girl group based in London, England?', 'supporting_facts': [['The Voice of Ireland (series 4)', 3], ['The Saturdays', 0]], 'context': [['List of Girls Aloud concert tours', ['British-Irish girl group Girls Aloud have embarked on six concert tours of the United Kingdom and Ireland, five of which were in arena-sized venues.']], ['Vanessa White', ['Vanessa Karen White (born 30 October 1989) is an English singer-songwriter, dancer and actress.', ' She rose to fame in 2008 as a member of British-Irish girl group The Saturdays, signed to Fascination and Polydor Records.', ' The group have achieved substantial success with numerous top-ten hits as well as a number one single.', ' She is the youngest member of the group.']], ['Forever Is Over', ['\"Forever Is Over\" is a single by British-Irish girl group The Saturdays.', ' The track was released by digital download on 3 October 2009, and CD single the following day.', ' The song was written by, Louis Biancaniello, Kahmarl Gordon, Sam Watters, James Bourne and produced by The Runaways.', ' It was released as the lead single from the group\\'s second studio album \"Wordshaker\".']], ['Headlines!', ['Headlines!', ' is the debut EP by British-Irish girl group The Saturdays.', ' It was released in Ireland on 13 August 2010 and in the United Kingdom on 16 August 2010 by Fascination Records.', ' The collection includes previous singles \"Forever Is Over\" and \"Ego\" as well as a remix of \"One Shot\" from \"Wordshaker\" and five new songs recorded in 2010.', ' One of these is a cover of the song \"Died in Your Eyes\", originally from Kristinia DeBarge\\'s 2009 album \"Exposed\".', ' The album was preceded by the release of the lead single \"Missing You\" on 5 August 2010.', \" It became the group's seventh top-ten hit when it peaked at number three in the UK and number six in Ireland.\"]], ['On Your Radar', ['On Your Radar is the third studio album by British-Irish girl group The Saturdays.', ' The album was released 21 November 2011 under Fascination Records.', ' The album was recorded in Los Angeles and London.', \" Steve Mac, who has been instrumental in the production of the band's previous albums, serves as a major collaborator.\", ' Additional producers and songwriters include Lucas Secon, Taio Cruz, Labrinth, Tracklacers, Space Cowboy, Brian Higgins and Lucie Silvas.']], ['Higher (The Saturdays song)', ['\"Higher\" is a song performed by British-Irish girl group The Saturdays taken from their debut extended play, \"Headlines!', '\".', \" Written by Ina Wroldsen and co-written & produced by Arnthor Birgisson, the song confirmed to be the EP's second single when it was released on 1 November 2010 by Fascination Records.\", ' In preparation for its release, the single was remixed to feature new vocals from American rapper Flo Rida after the rapper\\'s single \"Club Can\\'t Handle Me\" beat the group\\'s previous single \"Missing You\" to number one; it was the second time they had lost the position to Flo Rida.', ' Rochelle Wiseman jokingly said that the group would never get a number one until they collaborated with him.']], ['The Voice of Ireland (series 4)', ['The Voice of Ireland is an Irish reality talent show.', ' The fourth series of the programme premiered on 4 January 2015 on RTÉ One and concluded on 26 April, similar to previous seasons.', \" Kathryn Thomas returned as main host, while her colleague Eoghan McDermott remained the show's co-host.\", ' Television personalities Bressie and Kian Egan both returned as coaches, while The Saturdays singer Una Healy and S Club 7 star Rachel Stevens filled the remaining two positions.', ' Screentime Shinawil Productions continued to produce the talent programme.']], ['Notorious (The Saturdays song)', ['\"Notorious\" is a song by British-Irish girl group The Saturdays, released as the lead single from the third studio album, \"On Your Radar\".', ' The song was written by Ina Wroldsen and Steve Mac, who also produced the song.', ' The single was released digitally on 22 May 2011.', ' \"Notorious\" was recorded in 2011 after the band were looking for a new sound to release in their third album.', ' According to the lyrics in the song the band are loving life, they like to go out and enjoy themselves however they work hard and they all have a bad and naughty side to them.', ' The lyrics of the song has been described as \"playful\".', ' Vanessa White said the song was a more \"grown up\" approach to the music they have released before.', ' The group performed the song live for the first time on \"So You Think You Can Dance\" and the song received its first radio airplay on \"The Chris Moyles Show\".']], ['The Saturdays', ['The Saturdays was a British-Irish girl group based in London, England.', ' The group formed during the summer of 2007.', ' The line up consisted of Frankie Bridge, Una Healy, Rochelle Humes, Mollie King and Vanessa White.', ' They were formed through Fascination Records, who gave them an instant record deal with the label as well as a sub-division of Polydor Records.', ' As soon as the contract was finalised The Saturdays went on tour with Girls Aloud during their Tangled Up Tour.', \" The group's music style is pop, however throughout their career their management have experimented with dance-pop and electropop.\", ' To create this music, Ina Wroldsen, Steve Mac and Quiz & Larossi have been heavily involved.']], ['Issues (The Saturdays song)', ['\"Issues\" is a mid-tempo pop / R&B song performed by British-Irish girl group The Saturdays.', ' The song was written and produced by Carl Sturken and Evan Rogers.', ' The single was released as the band\\'s third official single, from their first studio album, \"Chasing Lights\".', ' \"Issues\" was officially released on 5 January 2009 in the United Kingdom.', ' The single gained to mixed reviews from pop music critics.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.680\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a79c8fa5542994bb94570a1', 'answer': 'technical director', 'question': 'What role did Veda Hlubinka-Cook have in the making of this 1997 video game that takes place on the Orient Express, days before the start of World War I?', 'supporting_facts': [['Veda Hlubinka-Cook', 2], ['The Last Express', 0], ['The Last Express', 1]], 'context': [['Mokujin', ['Mokujin (Japanese: 木人?', ', lit.', ' \"Wood(en) person\") is a fictional character in Namco Bandai Games\\' \"Tekken\" video game series.', ' Mokujin first appeared in the 1997 video game \"Tekken 3\".', ' Mokujin does not have his own fighting style.', ' Instead, he mimics fighting styles from other characters, which varies from match to match.']], ['Rother Valley Railway', ['The Rother Valley Railway (RVR) is a heritage railway project based at Robertsbridge in East Sussex, England.', ' It takes its name from the original name for what later became the Kent and East Sussex Railway, running from Robertsbridge through to Headcorn in Kent, via Tenterden.', ' The project is to replace the ‘missing link’ between Robertsbridge, a station on the Tonbridge to Hastings mainline, and Bodiam on the Kent and East Sussex Railway, a heritage railway which operates from Bodiam to Tenterden.', ' A charity supported by a society of volunteers are attempting to re-establish the railway link.', ' The RVR began by reinstating the first few hundred yards of line eastwards from Robertsbridge, and also a short stretch westwards from Bodiam.', ' In 2010, the latter section was further extended to reach Junction Road.', ' In summer 2011 work began at Robertsbridge to extend further eastwards to Northbridge Street, which entailed the rebuilding of five bridges.', ' By June 2012, this further extension was also completed.', ' In September 2013, a Gala weekend at Robertsbridge marked the progress to date and the start of the next phase - the re-instatement of the section between Northbridge Street and Junction Road, for which statutory permissions are being sought.', ' While the RVR does not yet feature regular passenger trains, the base at Robertsbridge houses a small shop and visitor centre open to the public each Sunday, utilising a building formerly used as the London terminus of the Orient Express.', ' There is also a small collection of historic railway vehicles in various stages of preservation.']], ['Before Crisis: Final Fantasy VII', ['Before Crisis: Final Fantasy VII (Japanese: ビフォア クライシス -ファイナルファンタジーVII- , Hepburn: Bifoa Kuraishisu -Fainaru Fantajī Sebun- ) is an action role-playing video game developed by Square Enix and originally released for the FOMA mobile service on September 24, 2004.', ' It was later released for SoftBank Mobile and EZweb in 2007.', ' \"Before Crisis\" is a prequel to the 1997 video game \"Final Fantasy VII\" and forms part of the \"Compilation of Final Fantasy VII\", a metaseries expanding on and continuing the story established in \"Final Fantasy VII\".', ' It takes place six years prior to the events of \"Final Fantasy VII\" and focuses on the adventures of the Turks, a group of covert operatives working for the Shinra Electric Power Company, and their fights against both rebel group AVALANCHE and their corrupt employers.']], ['Veda Hlubinka-Cook', ['Veda Hlubinka-Cook (born Robert Cook) is a co-founder of Metaweb.', ' She was a software programmer at Brøderbund in the 1980s and was the model for one of the characters in Jordan Mechner\\'s game \"Prince of Persia\".', ' She designed and created the computer game \"D/Generation\" and was technical director for the computer game \"The Last Express\".', ' She came out as transgender in 2017.']], ['Heavy Gear II', ['Heavy Gear II is a mecha based first-person shooter video game.', \" Set in Dream Pod 9's Heavy Gear universe, the game was developed and published by Activision in 1999 for Microsoft Windows, it was ported to Linux in 2000 by Loki Software.\", ' It is a sequel to the 1997 video game \"Heavy Gear\".']], ['Mob Rule', ['Mob Rule (also known as Constructor: Street Wars and Street Wars: Constructor Underworld) is a real-time strategy video game for PC released in 1999 by Simon & Schuster and Studio 3.', ' It is the successor to the 1997 video game \"Constructor\".', ' The goal of the game is to construct buildings and fight enemy teams in a Mafia-themed background.']], ['BioShock 2', ['BioShock 2 is a first-person shooter video game developed by 2K Marin and published by 2K Games.', ' It is the sequel to the 2007 video game \"BioShock\" and was released worldwide for Microsoft Windows, the PlayStation 3, and the Xbox 360 on February 9, 2010.', ' Feral Interactive released an OS X version of the game on March 30, 2012.', ' Set in the fictional underwater dystopian city of Rapture, the game\\'s story takes place eight years after the events of the first \"BioShock\".', ' Assuming control of Subject Delta, a hulking Big Daddy, players are tasked with fighting through \"splicers\", the psychotic human population of the city, using weapons and an array of genetic modifications.', ' The game also introduces a story-driven multiplayer mode called \"Fall of Rapture\", which takes place during Rapture\\'s 1959 civil war, before the events of the first game.']], ['The Last Express', ['The Last Express is an adventure video game designed by Jordan Mechner and published by Brøderbund in 1997.', ' It takes place on the Orient Express, days before the start of World War I.', ' It is noted as being one of the few video games that attempts to realistically simulate real time.', ' The game was a commercial disappointment, but received highly positive reviews and a positive post-release response.', ' A Sony PlayStation port was planned and was almost finished for release, but was cancelled for unknown reasons.']], ['Monopoly Star Wars', ['Monopoly Star Wars is a 1997 video game based on the board game of the same name.', ' It is one of many video game adaptions of the board game Monopoly.', ' The game was developed by Artech Digital Entertainment and published by Hasbro Interactive.', ' It was released exclusively for Microsoft Windows.', ' The game employs the same basic ruleset of traditional Monopoly gameplay, but adds a \"Star Wars\" theme which includes famous characters and locales in place of the original game pieces and properties.']], ['Aliens: Colonial Marines', ['Aliens: Colonial Marines is a first-person shooter video game developed by Gearbox Software and published by Sega.', ' The game is set in the \"Alien\" universe and takes place after the events of the third film in the franchise, \"Alien 3\" (1992) and takes place nearly 17 weeks after the events of \"Aliens\".', ' The game is the second \"Alien\" title that Sega has produced: the first being \"\" (2011).', ' The game was released for the PlayStation 3 and Xbox 360 video game consoles and for Microsoft Windows on February 12, 2013.', ' The game received overwhelmingly negative reviews from critics and is now considered to be one of the worst video games of all time, being especially panned for its poor story and glitchy enemy AI.', ' A Wii U version was being produced for release but was eventually cancelled on April 5, 2013.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.681\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae43d7f5542996836b02c59', 'answer': 'Barbara Albert', 'question': 'Which director, John Schlesinger or Barbara Albert, was also a writer and film producer?', 'supporting_facts': [['John Schlesinger', 0], ['Barbara Albert', 0]], 'context': [['Billy Liar (film)', ['Billy Liar is a 1963 British black-and-white CinemaScope comedy-drama film based on the 1959 novel by Keith Waterhouse.', ' Directed by John Schlesinger, it stars Tom Courtenay (who had understudied Albert Finney in the West End theatre adaptation of the novel) as Billy, and Julie Christie as Liz, one of his three girlfriends.', ' Mona Washbourne plays Mrs. Fisher, and Wilfred Pickles plays Mr. Fisher.', ' Rodney Bewes, Finlay Currie and Leonard Rossiter also feature.', ' The Cinemascope photography is by Denys Coop, and Richard Rodney Bennett supplied the score.']], ['Katharine Schlesinger', ['Katharine Schlesinger, is a British actress niece of the film director John Schlesinger and great-niece of Dame Peggy Ashcroft.', ' She starred as Catherine in the 1986 film adaptation of Jane Austen\\'s \"Northanger Abbey\".']], ['Joseph Janni', ['Joseph Janni (21 May 1916 – 29 May 1994) was a British film producer best known for his work with John Schlesinger.', ' He was born into a Jewish family in Milan, Italy and became interested in filmmaking while at university.', ' He emigrated to England in 1939, and once Italy declared war against England was briefly interned in Metropole Camp on the Isle of Man.', ' He soon became involved in the British film industry and worked his way up to producer.', ' He produced the first films of Schlesinger and Ken Loach.']], ['Leon Schlesinger', ['Leon Schlesinger (May 20, 1884 – December 25, 1949) was an American film producer, remembered for founding , which later became the Warner Bros.', ' Cartoons studio, during the Golden Age of American animation.', ' He was also a distant relative of the Warner Brothers.', ' As head of his own studio, Schlesinger served as the producer of Warner\\'s \"Looney Tunes\" and \"Merrie Melodies\" cartoons from 1930, when Schlesinger assumed production from his subcontractors, Harman-Ising, to 1944, when Warner acquired the studio.']], ['Far from the Madding Crowd (1967 film)', [\"Far from the Madding Crowd is a 1967 British epic drama film adapted from Thomas Hardy's book of the same name.\", \" The film, starring Julie Christie, Alan Bates, Terence Stamp and Peter Finch, and directed by John Schlesinger, was Schlesinger's fourth film (and his third collaboration with Christie).\", ' It marked a stylistic shift away from his earlier works exploring contemporary urban mores.', ' The cinematography was by Nicolas Roeg and the soundtrack was by Richard Rodney Bennett.', ' He also used traditional folk songs in various scenes throughout the film.']], ['Barbara Albert', ['Barbara Albert (born 1970 in Vienna) is an Austrian writer, film-producer and film-director.']], ['Peter Buckman', ['Peter Buckman is an English writer and literary agent.', ' He has been involved in the publishing industry for many years; he was on the editorial board of Penguin Books, and a commissioning editor for the New American Library in New York City.', ' He has published novels, non fiction, a biography and short stories, and has also written for television and film, including an episode of Inspector Morse, three episodes of The House of Elliot and television movie The Tale of Sweeney Todd, which was directed by John Schlesinger.', ' He is currently a literary agent, having started The Ampersand Agency in 2003, notable for having discovered and represented Vikas Swarup, author of Q & A, which was filmed as Slumdog Millionaire, and also being the agent for the estate of prolific historical fiction writer Georgette Heyer.']], ['Nat Cohen', ['Nat Cohen (23 December 1905 – 10 February 1988) was a British film producer and executive.', ' For over four decades he was one of the most significant figures in the British film industry, particularly in his capacity as head of Anglo-Amalgamated and EMI Films; he helped finance the first \"Carry On\" movies and early work of filmmakers such as Ken Loach, John Schlesinger, Alan Parker and David Puttnam.', ' In the early 1970s while head of EMI Films he was called the most powerful man in the British film industry.']], ['John Schlesinger', ['John Richard Schlesinger, CBE ( ; 16 February 1926 – 25 July 2003) was an English film and stage director, and actor.', ' He won an Academy Award for Best Director for \"Midnight Cowboy\", and was nominated for two other films (\"Darling\" and \"Sunday Bloody Sunday\").']], ['Lisa Eichhorn', ['Lisa Eichhorn (born February 4, 1952) is an American actress, writer and producer.', ' She made her film debut in 1979 in the John Schlesinger film \"Yanks\" for which she received two Golden Globe nominations.', ' Her international career has included film, theatre and television.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.681\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a862974554299211dda2a95', 'answer': 'John William Henry II', 'question': 'Who is older, Jed Hoyer or John William Henry II?', 'supporting_facts': [['Jed Hoyer', 0], ['John W. Henry', 0]], 'context': [['John W. Henry', ['John William Henry II (born September 13, 1949) is an American businessman and investor and the founder of John W. Henry & Company, an investment management firm.', ' He is the principal owner of \"The Boston Globe\", the Boston Red Sox and Liverpool Football Club and co-owner of Roush Fenway Racing.', ' In March 2006, \"Boston Magazine\" estimated Henry\\'s net worth at $1.1 billion but noted that his company had recently experienced difficulties.', \" In November 2012, the company announced that it would stop managing clients' money by the end of the year, and John Henry confirmed that total assets under the firm's management had fallen from $2.5 billion in 2006 to less than $100 million as of late 2012.\"]], ['Constitutions of Clarendon', ['The Constitutions of Clarendon were a set of legislative procedures passed by Henry II of England in 1164.', ' The Constitutions were composed of 16 articles and represent an attempt to restrict ecclesiastical privileges and curb the power of the Church courts and the extent of Papal authority in England.', \" In the anarchic conditions of Henry II's predecessor, Stephen, the church had extended its jurisdiction by taking advantage of the weakness of royal authority.\", \" The Constitutions were claimed to restore the judicial customs observed during the reign of Henry I (1100–35), while in fact they were a part of Henry II's larger expansion of royal jurisdiction into the Church and civil law, which was a defining aspect of his reign.\"]], ['Sacramentary of Henry II', ['The Sacramentary of Henry II (German: \"Sakramentar Heinrichs II.\")', ', also called the Regensburg Sacramentary (\"Regensburger Sakramentar\"), is a manuscript of liturgical texts, which was created in Regensburg at the order of Emperor Henry II (r. 995-1024).', ' It is among the most significant works of Ottonian illumination.', ' The manuscript was gifted to Bamberg Cathedral by Henry II, was part of the Cathedral treasury until 1803 when it became part of the Bavarian State Library as a result of Secularisation.', ' It remains there today, stored under the inventory number \"clm 4456\".', ' It is modelled on the Codex Aureus of St. Emmeram donated by Charles the Bald in 870.']], ['William Henry (brother of Patrick Henry)', ['William Henry (1734–1785) was the son of John and Sarah Winston Syme Henry.', ' He was the older brother of Patrick Henry, who is known for his famous \"Give me Liberty, or give me Death!', '\" speech.', ' William Henry lived in Virginia and served in the House of Burgesses.', ' He was elected to the Assembly as a member from Fluvanna County.']], ['Henry II style', ['The Henry II style was the chief artistic movement of the sixteenth century in France, part of Northern Mannerism.', ' It came immediately after High Renaissance and was largely the product of Italian influences.', \" Francis I and his daughter-in-law, Catherine de' Medici, had imported to France a number Italian artists of Raphael's or Michelangelo's school; the Frenchmen who followed them in working in the Mannerist idiom.\", ' Besides the work of Italians in France, many Frenchman picked up Italianisms while studying art in Italy during the middle of the century.', ' The Henry II style, though named after Henry II of France, in fact lasted from about 1530 until 1590 under five French monarchs, their mistresses and their queens.']], ['Eleanor, Fair Maid of Brittany', ['Eleanor Fair Maid of Brittany (c. 1184 – 10 August 1241), also known as Damsel of Brittany, Pearl of Brittany, or Beauty of Brittany, was the eldest daughter of Geoffrey II, Duke of Brittany, the fourth son of King Henry II of England, and Constance, Duchess of Brittany.', ' After the presumed death in 1203 of her imprisoned younger brother, Arthur, she was heiress to vast lands including England, Anjou, and Aquitaine as well as Brittany, realms where the Salic Law barring the accession of females did not apply.', \" Her uncle John, King of England was the fifth son of Henry II, and Eleanor inherited Arthur's claim to the throne as child of John's elder brother Geoffrey.\", ' Thus she posed a potential threat to John, and following his death in 1216, equally to her cousin, Henry III of England.', ' She was imprisoned from 1202, and thus became the longest-imprisoned member of an English royal family.', \" As a prisoner she was also unable to press her claim to the Duchy of Brittany as her mother's heiress.\"]], ['Grouseland', ['Grouseland, the William Henry Harrison Mansion and Museum, is a National Historic Landmark important for its architecture and role in history.', ' Grouseland is a large, two-story red brick home built for William Henry Harrison in Vincennes, Indiana, during his term as Governor of the Indiana Territory.', ' The mansion was completed in 1804 and reportedly dubbed \"Grouseland\" by William Henry Harrison due to the abundance of grouse in the area.']], ['Jed Hoyer', ['Jed D. Hoyer (born December 7, 1973), is the executive vice-president and general manager of the Chicago Cubs.', ' He has been the general manager of the San Diego Padres and the assistant general manager of the Boston Red Sox.', ' He joined the Red Sox in 2002, after the ownership of John W. Henry, Tom Werner, and Larry Lucchino took over the team from John Harrington.', ' He worked under the title of assistant to the general manager until December 2005.', ' He then was given the title of assistant general manager.', ' Hoyer briefly served as co-general manager of the Red Sox from December 12, 2005 to January 19, 2006 and then returning to his previous job of assistant general manager.', \" In November 2003, he accompanied general manager Theo Epstein to Arizona to persuade pitcher Curt Schilling to accept a trade to the Red Sox, spending Thanksgiving at Schilling's home in what was eventually a successful effort.\"]], ['Geoffrey Plantagenet, Count of Anjou', ['Geoffrey V (24 August 1113 – 7 September 1151) — called the Handsome or the Fair (French: \"le Bel\" ) and Plantagenet — was the Count of Anjou, Touraine, and Maine by inheritance from 1129 and then Duke of Normandy by conquest from 1144.', ' By his marriage to the Empress Matilda, daughter and heiress of Henry\\xa0I of England, Geoffrey had a son, Henry Curtmantle, who succeeded to the English throne as King Henry II (1154-1189) and was the first of the Plantagenet dynasty to rule England; the name \"Plantagenet\" was taken from Geoffrey\\'s epithet.', \" His ancestral domain of Anjou gave rise to the name Angevin for three kings of England (Henry II his son and heir, and Henry's sons Richard and John), and what became known as the Angevin Empire in the 12th century.\"]], ['Angevin kings of England', ['The Angevins (\"from Anjou\") were an English royal house in the 12th and early 13th centuries; its monarchs were Henry II, Richard I and John.', ' In the 10 years from 1144, two successive counts of Anjou, Geoffrey and his son, the future Henry II, won control of a vast assemblage of lands in western Europe that would last for 80 years and would retrospectively be referred to as the Angevin Empire.', ' As a political entity this was structurally different from the preceding Norman and subsequent Plantagenet realms.', ' Geoffrey became Duke of Normandy in 1144 and died in 1151.', ' In 1152 his heir, Henry, added Aquitaine by virtue of his marriage to Eleanor of Aquitaine.', ' Henry also inherited the claim of his mother, Empress Matilda, the daughter of King Henry I, to the English throne, to which he succeeded in 1154 following the death of King Stephen.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.681\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7769a35542993569682d8f', 'answer': 'The Ten-Year Lunch', 'question': 'Which documentary was release first, Baghdad ER or The Ten-Year Lunch?', 'supporting_facts': [['Baghdad ER', 0], ['The Ten-Year Lunch', 0]], 'context': [['Antenna (GO!GO!7188 album)', ['Antenna (アンテナ , \"Antenna\" ) is the seventh studio album by Japanese rock band GO!GO!', '7188.', ' The limited release first press also included a DVD featuring PV\\'s for the single \"Futashika Tashika\" and a live performance, \"Omata Kara no Live Eizou.\"']], ['2007 Iraqi Parliament bombing', ['On 12 April 2007, the canteen of the Council of Representatives of Iraq building was attacked by a suicide bomber, killing one to eight people and wounding 23 others.', ' The attack, in the heavily fortified Green Zone of Baghdad, occurred ten minutes after the Council of Representatives had adjourned for lunch.', ' It was on the first floor of the Baghdad Convention Center, which houses the parliament.', ' Two further unexploded suicide vests were found near the canteen.', ' The building had earlier been searched by dogs – very rare considering dogs are considered ritually unclean by Iraqis – suggesting the authorities suspected an attack was imminent.', ' Following the attack the government closed down mobile phone networks and Apache helicopters flew overhead.']], ['About Baghdad', ['About Baghdad is a documentary film shot in Baghdad, Iraq in 2003.', ' It is the first documentary film to have been made in Iraq following the fall of the Baath regime.', ' The film features the artist Sinan Antoon as he returns to his native Baghdad.', \" It privileges the voices of native Iraqis from all walks of life, as they present their views on life during the regime of Saddam Hussein as well as the United States's bombing, invasion, and occupation.\"]], ['Anant Nag filmography', ['Anant Nag is an Indian film actor and an occasional film producer who appears as an actor in Kannada, Hindi, Telugu, Marathi, Malayalam and Tamil films, but predominantly in Kannada films.', ' In a career spanning over 40 years, he has appeared in over 220 films.', ' After having had a successful theatre career, he made his debut in P. V. Nanjaraja Urs\\' Kannada film \"Sankalpa\", and Shyam Benegal\\'s Dakhani film \"Ankur\", with the former seeing theatrical release first, in 1973, and won multiple awards at the 1972–73 Karnataka State Film Awards.', ' In G. V. Iyer\\'s 1975 film \"Hamsageethe\", he played the role of Bhairavi Venkatasubbiah, a performance that won critical praise, and the film was awarded the Best Feature Film in Kannada at the 23rd National Film Awards.']], ['Flash Best', ['Flash Best is the first compilation album by the Japanese electronica band Capsule.', ' The limited release first press also included a DVD with music videos of \"Flash Back\", \"Jumper\",\\u3000\"Sugarless Girl\", \"Glider\", \"Portable Airport\",\\u3000\"Space Station No.9\" and\\u3000\"Soratobu Toshikeikaku\".']], [\"Matthew O'Neill (filmmaker)\", ['Matthew O\\'Neill is a documentary filmmaker best known for his work on the HBO film \"Baghdad ER\", for which he and co-creator Jon Alpert won three Emmy Awards.']], ['569 (album)', ['569 (ゴーロック , \"Gō Rokku\" ) is the sixth studio album by Japanese rock band GO!GO!', '7188.', ' The title is a play on words with the Japanese pronunciation of \"569\" sounding like the English \"Go Rock You\".', ' The limited release first press also included a DVD featuring video highlights of their first foreign tour in the United States in March 2007.']], ['Finder no Mukou', ['Finder no Mukou (ファインダーの向こう , Faindā no mukō ) is the third studio album by Japanese singer Shiori Niiyama.', ' It was released on 30 November 2016, one year and five months after second studio album Hello Goodbye.', ' The album was recorded under Being Inc. label.', ' Album includes previous 2 released singles- \"Tonari no Yukue\" and \"Atashi wa Atashi no Mama de\".', ' A famous Japanese musicians as Fukuyama Masaharu were involved with the music production of album.', ' The album consists of three version: regular one with special CD of coupling songs, first press release first version which includes special DVD disc with music clips and second version with live performances.', ' The album reached #14 in daily rank and #26 for first week.', \" It's charting for two weeks.\"]], ['Baghdad ER', ['Baghdad ER is a documentary released by HBO on May 21, 2006.', ' It shows the Iraq war from the perspective of a military hospital in Baghdad.', ' It has some relatively disturbing scenes in it (e.g. amputations), therefore the U.S. Army is officially warning that military personnel watching it could experience symptoms of post-traumatic stress disorder (PTSD).']], ['The Ten-Year Lunch', ['The Ten-Year Lunch: The Wit and Legend of the Algonquin Round Table is a 1987 American documentary film about the Algonquin Round Table, a floating group of writers and actors in the \"Roaring Twenties\" in New York City, which included great names such as Dorothy Parker, Robert Benchley, George S. Kaufman, Edna Ferber, Marc Connelly, Harold Ross and Harpo Marx.', ' It was produced and directed by Aviva Slesin and narrated by Heywood Hale Broun.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.683\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab54e145542992aa134a2a1', 'answer': 'Kentucky, Virginia, and Tennessee', 'question': 'What three states junction is near the gap that Peter Hackett traveled along during the Scotish-Irish migration along the Wilderness Road?', 'supporting_facts': [['Peter Hackett', 2], ['Cumberland Gap', 0]], 'context': [['Culture of Oregon', ['The culture of Oregon has had a diverse and distinct character from before European settlement until the modern day.', ' Approximately 80 Native American tribes were living in Oregon before the establishment of European pioneer settlements.', ' Trappers and traders were the harbingers of the coming migration of Europeans.', ' Many of these settlers traveled along the nationally renowned Oregon Trail, with estimates of around 53,000 using the trail between 1840 and 1850.']], ['Wilderness Road State Park', ['Wilderness Road State Park is a state park located in southwestern Virginia, near Cumberland Gap and Ewing, VA.', ' It consists of about 310 acre around the former Wilderness Road.']], ['O. C. Hackett', ['Oliver Cromwell Hackett was born March 29, 1822 in Scott County, Kentucky.', ' His father was John Hackett, and his grandfather was noted Kentucky frontiersman and militiaman of the American Revolution, Peter Hackett.', ' John Hackett moved the family, including young O. C., from Kentucky to Coles County, Illinois in 1835.', ' O. C. Hackett married Ellen Roxanne (Wyeth) on March 14, 1854.', \" O. C.'s children included Frederick W. Hackett.\", ' O. C. died April 8, 1905 in Tuscola, Illinois.', ' Family legend holds that Abraham Lincoln stayed at the Hackett farm near Charleston Illinois before or after the 4th of the Lincoln-Douglas debates of 1858.']], ['Fincastle Turnpike', ['The Fincastle Turnpike, also known as the \"Fincastle and Blue Ridge Turnpike Company\", was approved in 1834 by the Virginia General Assembly to maintain a toll turnpike that followed part of the Wilderness Road from Fincastle, Virginia, to the Cumberland Gap.', ' The Fincastle Turnpike also connected Narrows, Virginia and Tazewell, Virginia along the way to the Cumberland Gap, following roughly what is today parts of Virginia State Route 42 and Virginia State Route 61.']], ['Peter Hackett', ['Peter Hackett was born in approximately 1763 or 1764 in the English colony of Virginia.', ' It is believed that Peter was the son of Thomas Hackett, likely of Montgomery County, Virginia.', ' As a boy Peter was bonded out to Captain James Estill, in approximately 1771, and was a part of the broad Scotch-Irish migration along the Wilderness Road through the Cumberland Gap from Virginia into what later became known as Kentucky in the late 18th century.', ' In 1779 he was a resident of Boonesborough, one of the first English-speaking settlements beyond the Appalachian Mountains, and lived there until 1780.', ' Early residents of Boonesborough included Daniel Boone, James Estill, Joseph Proctor, Nicholas Proctor, Adam Caperton, David Lynch, John Colefoot, John Moore, George Robertson, Thomas Miller, Reuben Proctor, Thomas Warren, Peter Hackett, and Thomas Watson.', \" In 1780 Hackett helped establish Estill's Station, Kentucky, and lived there until about 1788.\"]], ['2011 Philadelphia, Mississippi tornado', ['During the afternoon of April\\xa027, 2011, a violent EF5 tornado touched down in eastern Mississippi, killing three people.', ' Part of the historic 2011 Super Outbreak, the largest tornado outbreak on record, this was the first of four EF5 tornadoes to touch down that day and the first such storm in Mississippi since the 1966 Candlestick Park tornado.', ' While on the ground for 30\\xa0minutes, it traveled along a near 29 mi path through four counties, leaving behind three deaths, eight injuries, and $1.1\\xa0million in damage.']], ['Tropical Storm Wukong (2006)', ['Severe Tropical Storm Wukong was a slow moving tropical cyclone which produced torrential rains over Japan.', ' The tenth named storm of the 2006 Pacific typhoon season, Wukong developed out of a tropical depression over the open waters of the western Pacific Ocean.', ' On August\\xa013, both the Japan Meteorological Agency (JMA) and the Joint Typhoon Warning Center (JTWC) classified the depression as a tropical storm.', ' The storm traveled along a curving path south of Japan, absorbing the remnants of Tropical Storm Sonamu on August\\xa015 before turning towards the west.', ' Wukong made landfall at peak intensity late on August\\xa017 near Miyazaki City in southern Kyūshū.', ' The cyclone remained over land for about 24\\xa0hours before moving out over the Sea of Japan.', ' The storm weakened to a tropical depression before dissipating on August\\xa021.', ' Due to the slow movement of the storm, it produced heavy rains, peaking at 516\\xa0mm (20.3\\xa0in).', ' Two people were killed due to rough seas produced by the storm and three others were injured.']], ['Wilderness Road', ['The Wilderness Road was one of two principal routes used by colonial and early national era settlers to reach Kentucky from the East.', ' Although this road goes through the Cumberland Gap into southern Kentucky and northern Tennessee, the other (more northern route) is sometimes called the \"Cumberland Road\" because it started in Fort Cumberland in Maryland.', \" Despite Kentucky Senator Henry Clay's advocacy of this route, early in the 19th century, the northern route was selected for the National Road, connecting near Washington, Pennsylvania into the Ohio Valley of northern Kentucky and Ohio.\"]], ['Cumberland Gap', ['The Cumberland Gap is a narrow pass through the long ridge of the Cumberland Mountains, within the Appalachian Mountains, near the junction of the U.S. states of Kentucky, Virginia, and Tennessee.']], ['Belgian railway line 130A', ['The Belgian railway line 130A is a railway line in Belgium connecting Charleroi with the French border near Erquelinnes.', ' Completed in 1852, the line runs 29.3\\xa0km.', ' It runs along the river Sambre, crossing it several times.', ' Beyond Erquelinnes, a French railway line continues towards Jeumont, Saint-Quentin and Paris.', ' Until the opening of the Paris–Brussels–Cologne high-speed lines, international passenger trains between Paris and Cologne traveled along line 130A.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.684\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5aba8d4655429901930fa82b', 'answer': '\"Woody\" Allen', 'question': 'Which writer, Karen Blixen or Woody Allen, has a broader range of artistic talents?', 'supporting_facts': [['Karen Blixen', 0], ['Woody Allen', 0]], 'context': [['Karen Blixen', ['Baroness Karen Christenze von Blixen-Finecke (née Dinesen; 17 April 1885 – 7 September 1962) was a Danish author who wrote works in Danish and English.', ' She is best known under her pen names Isak Dinesen, used in English-speaking countries, and Tania Blixen, used in German-speaking countries.', ' She also published works using the aliases Osceola and Pierre Andrézel.']], ['Karen Blixen Museum (Kenya)', ['The Karen Blixen Museum, located 10\\xa0km outside of Nairobi, Kenya, \"at the foot of the Ngong Hills\", is the former African home of Danish author Karen Blixen, famous for her book \"Out of Africa\" which chronicles life at the estate.']], ['Anecdotes of Destiny', ['Anecdotes of Destiny is a collection of stories by Danish author Karen Blixen.', \" It was the last work put out during Karen Blixen's lifetime; it was published in Denmark on October 12, 1958.\"]], ['Familien de Cats', ['Familien de Cats (The de Cats Family) was the third short story written by the Danish writer Karen Blixen under the pen name Osceola, a famous early 19th-century Native American leader.', ' Published in the literary journal \"Tilskueren\" in January 1909, it followed \"Eneboerne\" (The Hermits) and \"Pløjeren\" (The Ploughman), both published in 1907.', ' It tells the story of a law-abiding family which in each generation has a black sheep who turns out to be its secret blessing.', \" Although Blixen, then Karen Dinesen, wrote hundreds of pages of poems, essays and stories in her youth, it was only when she was 22 that she began to publish some of her short stories in literary journals as Osceola which had also been the name of her father's dog.\", ' Like her other early short stories, at the time it failed to attract attention.']], ['Bonnie S. Dunbar', ['Bonita Sue \"Bonnie\" Dunbar (born February 14, 1948) is a former professor in the department of molecular and cell biology at Baylor College of Medicine, a position she held from 1994 to 2004.', ' Prior thereto she was an assistant professor in the same department at the same university from 1981 to 1983.', ' From 1984 to 1994, also at Baylor College of Medicine, she also held a position as associate professor in the department of obstetrics and gynecology.', ' She is currently an honorary lecturer at the University of Nairobi.', ' She is a member of the American Association for the Advancement of Science, the Endocrine Society, the American Society for Cell Biology, and the New York Academy of Sciences.', ' She is perhaps best known for her work regarding the zona pellucida of mammalian eggs, and has written chapters of textbooks on the topic.', ' She has been called \"the new Karen Blixen,\" and is currently the owner of the Karen Blixen Coffee Garden Restaurant and Cottages, as well as the treasurer of the African Biomedical Center.', ' She also served on the editorial board of the journal Medical Veritas, which was published from 2004 to 2008 and endorsed anti-vaccine views.']], ['Woody Allen', ['Heywood \"Woody\" Allen (born Allan Stewart Konigsberg; December 1, 1935) is an American filmmaker, writer, actor, comedian, and musician whose career spans more than six decades.']], ['Out of Africa', ['Out of Africa is a memoir by the Danish author Karen Blixen.', ' The book, first published in 1937, recounts events of the seventeen years when Blixen made her home in Kenya, then called British East Africa.', ' The book is a lyrical meditation on Blixen’s life on her coffee plantation, as well as a tribute to some of the people who touched her life there.', ' It provides a vivid snapshot of African colonial life in the last decades of the British Empire.', ' Blixen wrote the book in English and then rewrote it in Danish.', \" The book has sometimes been published under the author's pen name, Isak Dinesen.\"]], ['Rungstedlund Award', ['The Rungstedlund Award is an award of honor, founded by the Rungstedlund Foundation in 1991.', ' The DKK 25,000 prize is annually handed to a person who has made a notable contribution in an area which interested Karen Blixen.', ' The prize is handed at the birthday of Karen Blixen on 17 April.', ' The award comes from a gift from Hørsholm Municipality at the opening of the Karen Blixen Museum on 14 May 1991.', ' Rungstedlund was owned by Karen Blixen from 1939-58.']], ['Rungstedlund', ['Rungstedlund, also known as the Karen Blixen Museum, is a country house in Rungsted on the Øresund coast just north of Copenhagen, Denmark, notable for its association with the author Karen Blixen, who lived there for most of her life.', ' She was born on the estate in 1885, and returned there after her years in Kenya, chronicled in her book \"Out of Africa\", to do most of her writings.', \" The property is today managed by the Rungstedlund Foundation as a writer's house museum.\"]], [\"Babette's Feast\", ['Babette\\'s Feast (Danish: \"Babettes gæstebud\" ) is a 1987 Danish drama film directed by Gabriel Axel.', \" The film's screenplay was written by Axel based on the story by Isak Dinesen (Karen Blixen).\", ' Produced by Just Betzer, Bo Christensen, and Benni Korzen with funding from the Danish Film Institute, \"Babette\\'s Feast\" was the first Danish cinema film of a Blixen story.', ' It was also the first Danish film to win the Academy Award for Best Foreign Language Film.', ' The film premiered in the Un Certain Regard section of the 1987 Cannes Film Festival.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.684\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ade157e55429939a52fe802', 'answer': 'Czech Kingdom', 'question': 'Jaroslav Kvapil the Czech poet was born in the Kingdom of Bohemia, also referred to by what name?', 'supporting_facts': [['Jaroslav Kvapil', 0], ['Kingdom of Bohemia', 0]], 'context': [['Ivan Wernisch', ['Ivan Wernisch (born 18 June 1942) is a Czech poet, editor and a collage artist.', ' He studied Ceramics Secondary school in Carlsbad (he left in 1959) and has since done many jobs, mostly manual.', ' In 1961, after publishing his debut poetry book, he quickly established himself as one of the best and most loved writers of his generation.', ' During the 70s and 80s he prepared many radio shows about famous poets of the world (in which he often – true to his interest in mystifications – wrote many of the poems himself), but his books could not be published officially.', ' After the Velvet revolution he worked in a newspaper.', ' Now he works as an editor in the Current Czech Poetry Library.', ' He is also a renowned translator from German, Dutch, Italian, Latin, French and Russian.', ' His work as an editor is focused mainly on forgotten Czech poets of the last three centuries.', ' Another Czech poet, Ewald Murrer, is his son.', ' Ivan Wernisch lives in Prague.']], ['Glagolitic Mass', ['The Glagolitic Mass (Czech: \"Glagolská mše\" or Mša glagolskaja; also called Missa Glagolitica or Slavonic Mass) is a composition for soloists (soprano, contralto, tenor, bass), double chorus, organ and orchestra by Leoš Janáček.', ' The work was completed on 15 October 1926 and premiered by the Brno Arts Society, conducted by Jaroslav Kvapil, in Brno on 5 December 1927.', ' Janáček revised the mass the next year.']], ['Saint Ludmila (oratorio)', ['Antonín Dvořák composed his oratorio Saint Ludmila (Czech: Svatá Ludmila (\\xa0\\xa0 ) for soloists, choir and orchestra, between September 1885 and May 1886.', ' The oratorio (Op. 71, B. 144) was written to a text by the leading Czech poet and writer Jaroslav Vrchlický.', ' \"Saint Ludmila\" is Dvořák\\'s third oratorio, and is considered one of his foremost works.']], ['Petr Mikeš', ['Petr Mikeš (August 19, 1948 Zlín, Czechoslovakia – February 8, 2016 Benešov, Czech Republic) was a Czech poet, translator, and editor.', ' In the 1970s and 1980s he took part in the samizdat edition \"Texty přátel\" (Texts of Friends).', ' From 1993–1997 he was the influential editor-in-chief of the Moravian publishing house Votobia, and from 2000–2004 at the Periplum publishing house (and co-founder: he took its name from a line by Ezra Pound).', ' He was a significant translator of Ezra Pound into Czech (he translated four generations of the Pound family into Czech: Homer Pound, Ezra Pound, Mary de Rachewiltz, and Patrizia de Rachewiltz).', ' He translated members of Pound\\'s \"circle\", including Basil Bunting, T.E. Hulme, and James Joyce, and even wrote a screenplay for a biopic on the life of Ezra Pound, \"Solitary Volcano\" (unproduced).']], ['Rusalka (opera)', ['Rusalka (] ), Op.', \" 114, is an opera ('lyric fairy tale') by Antonín Dvořák.\", ' The Czech libretto was written by the poet Jaroslav Kvapil (1868–1950) based on the fairy tales of Karel Jaromír Erben and Božena Němcová.', ' A Rusalka is a water sprite from Slavic mythology, usually inhabiting a lake or river.', ' \"Rusalka\" is one of the most successful Czech operas, and represents a cornerstone of the repertoire of Czech opera houses.']], ['Hana Janků', ['Hana Janků (25 October 1940 – 28 April 1995) was a Czech operatic soprano of international renown.', ' Born in Brno, she studied with Jaroslav Kvapil in her home city before making her professional opera début at the Brno Opera in Vítězslav Novák\\'s \"Lucerna\".', ' She became a principal singer at the Opéra national du Rhin and the Deutsche Oper am Rhein.', ' She made her La Scala début in 1967 and at the Deutsche Oper Berlin in 1970.', ' She also worked as a guest artist with several other major opera houses, including the Vienna State Opera, the Hamburg State Opera, and the Teatro Colón.', ' She was particularly admired for her portrayal of the title role in Giacomo Puccini\\'s \"Turandot\".', ' She died in Vienna.']], ['Jaroslav Kvapil', ['Jaroslav Kvapil (25 September 1868 in Chudenice, Kingdom of Bohemia – 10 January 1950 in Prague) was a Czech poet, playwright, and librettist.', ' From 1900 he was a director and Dramaturg at the National Theatre in Prague, where he introduced plays by Anton Chekhov, Henrik Ibsen and Maxim Gorky into the repertory.', ' Later he was a director at the Vinohrady Theatre (1921–1928).', ' He wrote six plays, but is today chiefly remembered as the librettist of Antonín Dvořák\\'s \"Rusalka\".']], ['Ivan Martin Jirous', ['Ivan Martin Jirous (23 September 1944 – 10 November 2011) was a Czech poet, best known for being the artistic director of the Czech psychedelic rock group The Plastic People of the Universe and later one of the organizers of the Czech underground during the communist regime.', ' He is also known more frequently as Magor, which can be roughly translated as \"loony\" or \"fool\" and is supposedly derived from \"phantasmagoria\".', ' This nickname was given to him by the \"experimental\" poet Eugen Brikcius.', \" His wife, Věra Jirousová, wrote a good deal of the Plastics' early lyrics.\"]], ['Kingdom of Bohemia', ['The Kingdom of Bohemia, sometimes in English literature referred to as the Czech Kingdom (Czech: \"České království\" ; German: \"Königreich Böhmen\" ; Latin: \"Regnum Bohemiae\" , sometimes Latin: \"Regnum Czechorum\" ), was a medieval and early modern monarchy in Central Europe, the predecessor of the modern Czech Republic.', ' It was an Imperial State in the Holy Roman Empire, and the Bohemian king was a prince-elector of the empire.', ' The kings of Bohemia, besides Bohemia, ruled also the Lands of the Bohemian Crown, which at various times included Moravia, Silesia, Lusatia and parts of Saxony, Brandenburg and Bavaria.']], ['Jaroslav Kvapil (composer)', ['Jaroslav Kvapil (21 April 1892 – 18 February 1958) was a Czech composer, teacher, conductor and pianist.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.685\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ade4da055429939a52fe878', 'answer': '1985', 'question': 'My Neighbor Totoro was produced by a Japanese animation film studio founded in what year?', 'supporting_facts': [['My Neighbor Totoro', 0], ['Studio Ghibli', 2]], 'context': [['Short films by Studio Ghibli', ['Studio Ghibli is a Japanese animation film studio founded in 1985.', ' In addition to producing 18 feature films, the studio has produced several short films, including commercials, films for the Ghibli Museum, music videos, and works released directly to video.']], ['Ghibli Museum', ['The Ghibli Museum (三鷹の森ジブリ美術館 , Mitaka no Mori Jiburi Bijutsukan , Mitaka Forest Ghibli Museum) is a museum showcasing the work of the Japanese animation studio Studio Ghibli.', ' It is located in Inokashira Park in Mitaka, a western city of Tokyo, Japan.', \" The museum combines features of a children's museum, technology museum, and a fine arts museum, and is dedicated to the art and technique of animation.\", ' Some features include a replica of the Catbus from \"My Neighbor Totoro\" (1988), a café, bookstore, rooftop garden, and a theater for exclusive short films by Studio Ghibli.']], ['Studio Ghibli', ['Studio Ghibli, Inc. (Japanese: 株式会社スタジオジブリ , Hepburn: Kabushiki-gaisha Sutajio Jiburi ) is a Japanese animation film studio based in Koganei, Tokyo, Japan.', ' The studio is best known for its anime feature films, and has also produced several short films, television commercials, and one television film.', ' It was founded on 15 June 1985, after the success of \"Nausicaä of the Valley of the Wind\" (1984), with funding by Tokuma Shoten.']], ['List of Digimon Adventure 02 episodes', ['\"Digimon Adventure 02\" is a 50-episode sequel of the 1999 anime series \"Digimon Adventure\".', ' It was created by Toei Animation and aired in Japan on Fuji TV between April 2, 2000, and March 25, 2001.', ' The series was directed by Hiroyuki Kakudō and produced by Keisuke Okuda.', ' Music for \"Digimon Adventure 02\" was composed by Takanori Arisawa, and characters were designed by Katsuyoshi Nakatsuru.', ' The story, set in an alternate timeline of the real world, opens four years after the events of \"Digimon Adventure\" with the next generation of DigiDestined children.', ' In their quest to maintain peace in the Digital World, the children battle both new and returning foes.', ' In a 2001 survey published by Japanese anime and entertainment magazine \"Animage\" of its readers, \"Digimon Adventure 02\" placed 17th, tied with the 1988 film \"My Neighbor Totoro\", on the list of anime that should be most remembered in the 21st century.']], ['First Human Giatrus', ['Giatrus (Japanese: ギャートルズ , Hepburn: Gyātoruzu ) is a Japanese manga written and illustrated by Shunji Sonoyama.', ' It spawned two other manga, two anime television series, a television drama, and an anime film.', ' The first TV series mark the debut of Joe Hisaishi, composer of \"My Neighbor Totoro\" and \"Spirited Away\".', ' The official English title is Gon, The Stone-Age Boy.']], ['Azumi Inoue', ['Azumi Inoue (井上 あずみ or 井上杏美 , Inoue Azumi , born February 10, 1965 in Kanazawa, Ishikawa Prefecture, Japan) is a Japanese singer.', ' She graduated from Yugakkan High School in Kanazawa.', ' She is best known for singing the opening and ending theme songs for the Hayao Miyazaki film \"My Neighbor Totoro\": \"Sanpo\" and \"My Neighbor Totoro\".', ' She is known for having a clear, light voice.']], ['My Neighbor Totoro', ['My Neighbor Totoro (Japanese: となりのトトロ , Hepburn: Tonari no Totoro ) is a 1988 Japanese animated fantasy film written and directed by Hayao Miyazaki and produced by Studio Ghibli.', ' The film – which stars the voice actors Noriko Hidaka, Chika Sakamoto, and Hitoshi Takagi – tells the story of the two young daughters (Satsuki and Mei) of a professor and their interactions with friendly wood spirits in postwar rural Japan.', ' The film won the Animage Anime Grand Prix prize and the Mainichi Film Award and Kinema Junpo Award for Best Film in 1988.', ' It also received the Special Award at the Blue Ribbon Awards in the same year.']], ['Makiko Futaki', ['Makiko Futaki (June 19, 1958 – May 13, 2016) was a Japanese animator best known for her work at Studio Ghibli for more than thirty years.', ' Futaki, who joined Studio Ghibli in 1981, worked on all of Hayao Miyazaki\\'s animated feature films, beginning with \"Nausicaä of the Valley of the Wind\" in 1984.', ' Her best known Studio Ghibli\\'s productions include \"My Neighbor Totoro\" (1988), \"Princess Mononoke\" (1997), \"Spirited Away\" (2001), which won an Academy Award for Best Animated Feature, and \"Howl\\'s Moving Castle\" (2004).', ' Her last film credit was Hiromasa Yonebayashi\\'s \"When Marnie Was There\" (2014), which is Studio Ghibli\\'s final feature film to date.']], ['Ufotable', ['Ufotable, Inc. (ユーフォーテーブル有限会社 , Yūfōtēburu yūgen-gaisha ) is a Japanese animation studio founded in October 2000 by former TMS Entertainment staff through its subsidiary Telecom Animation Film and located in Nakano, Tokyo Prefecture.', ' A unique hallmark seen in many of their works (\"Ninja Nonsense\", \"Futakoi Alternative\", \"Coyote Ragtime Show\", \"Gakuen Utopia Manabi Straight!', '\", \"\", \"Kara no Kyōkai\") is a claymation sequence.']], ['CoMix Wave Films', ['CoMix Wave Films, Inc. (Japanese: コミックス・ウェーブ・フィルム , Hepburn: Komikkusu Uēbu Firumu ) is a Japanese animation film studio and distribution company based in Chiyoda, Tokyo, Japan.', ' The studio is known for its anime feature films, short films, and television commercials, particularly those made by director Makoto Shinkai.', ' It was founded in March 2007 when it split from CoMix Wave Inc., which was initially formed in 1998 from Itochu Corporation, ASATSU (now ADK), and other companies.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.686\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae14b0655429920d52343a6', 'answer': 'Ramanaidu Daggubati', 'question': 'Bangalore Naatkal starred which actor and photographer?', 'supporting_facts': [['Bangalore Naatkal', 2], ['Rana Daggubati', 0]], 'context': [['Rana Daggubati', ['Ramanaidu Daggubati (born 14 December 1984), also known by his stage name Rana Daggubati or simply, Rana, is an Indian actor, producer, visual effects co-ordinator and photographer.', ' He is known for his works in Telugu cinema, Tamil cinema and Hindi cinema.', ' As a Visual Effects producer, Rana won the State Nandi Award for Best Special Effects in 2006 for the Telugu film \"Sainikudu\" starring Mahesh Babu.', ' In 2006, he received the National Film Award for co-producing \"Bommalata – A Bellyful of Dreams.\"']], ['Bangalore Naatkal', ['Bangalore Naatkal (English: \"Bangalore Days\" ) is a 2016 Indian Tamil comedy-drama film directed by Bommarillu Bhaskar, which is a remake of the 2014 Malayalam film \"Bangalore Days\" written and directed by Anjali Menon.', ' Featuring an ensemble cast consisting of Arya, Bobby Simha and Sri Divya in the lead roles, it tells the story of three cousins who live their childhood dream of living and enjoying in the city of Bangalore.', ' It also stars Rana Daggubati, Raai Laxmi, Parvathy and Samantha in other pivotal roles.', ' Produced by Prasad V Potluri under his banner PVP cinema, the film released on 5 February 2016.', ' Unlike the original, this movie received mixed reviews and failed at the box office, with critics and fans drawing comparisons with the original.']], ['Ankur Betageri', ['Ankur Betageri (born 18 November 1983 in Bangalore, Karnataka) is an Indian poet, fiction writer, photographer and arts activist.', ' He currently teaches English at Bharati College, University of Delhi.', ' In 2012, he was named as one of the ten best writers in the country by the English daily Indian Express.', ' He holds a Masters in Clinical Psychology from Christ University, Bangalore.', ' Betageri is also known for founding the public arts and activist platform, Hulchul, whose artistic interventions in reclaiming Public Spaces like public washrooms and roadside walls, and the use of art to transform the everyday urban life have been widely appreciated.', ' As a poet he has represented India at The III International Delphic Games (2009) at Jeju, South Korea, and Lit Up Writers Festival (2010) at Singapore.']], ['Ranga Shankara', [\"Ranga Shankara is one of Bangalore's well known theatres.\", ' It is located in the south Bangalore area of J.P Nagar and is run by the Sanket Trust.', ' The auditorium, which opened in 2004, was envisioned by Arundathi Nag, in remembrance of her late husband, Shankar Nag, who was a renowned actor in the Kannada film industry.']], ['K. S. Bhavani Shankar', ['K. S. Bhavani Shankar (born 16 October 1965), is an Indian Mridangam player, music composer and film actor known for his accompanying skills for Indian classical music and Dance.', ' He was chosen Best Accompanying artist in 1983, from Bangalore Gayana Samaj.', ' He was conferred the title \"Mridanga Chatura\" in 1984 by the Bangalore Gayana Samaj.', ' In 1996, he was awarded an Arts Council of England travel grant, the highest award given to traditional artists and musicians to perform outside UK.']], ['Antha Ezhu Naatkal', ['Antha 7 Naatkal (read as \"\"Antha Ezhu Naatkal\"\"; English: \"Those Seven Days\" ) is a 1981 Tamil language film.', ' Directed by and starred in by K. Bhagyaraj, the movie follows the life and ambitions of the hero, Palakkad Madhavan played by K. Bhagyaraj and his sidekick Gopi (child actor Khaja Sharif).', ' The movie was a great hit and very well received by the public.', ' It was remade in to the Telugu as \"Radha Kalyanam\" and then made in Hindi as \"Woh Saat Din\".']], ['Jim Ankan Deka', ['Jim Ankan Deka (; born 1 June 1980) is an Assamese musician, documentary film maker, photographer and director of Bangalore based organisation and music school Eastern Fare Music Foundation.', ' He is the first Assamese to open a music institute and a production house in Bangalore, India.', ' He won multiple awards for his song \"Aawaz - speak up against sexual violence\" based on the 2012 Delhi gang rape incident.']], ['PeeVee', ['Perumal Venkatesan aka PeeVee (1979, Villupuram, Tamil Nadu, India) is a people photographer and creative entrepreneur based in Bangalore, India.', ' He co-founded the Thalam website.', ' a creative space, both in Bangalore.', ' His photography works have been published in various Indian and international media.', ' His work has also been exhibited at art galleries across India.']], ['Saad Khan', ['Saad Khan, born in Mumbai, India, is an Indian film director, screenwriter, acting teacher, founder and creative head of Centerstage, a new wave media movement based in Bangalore that promotes new artists in the city.', ' He has worked in Bollywood as an Associate Director to Oscar nominated filmmaker Ashutosh Gowariker.', ' Saad made Bangalore\\'s first Hindi feature film \"\", independently produced by Sumit Ghosh and released across India by PVR Director\\'s Rare as well as in Indonesia; the film received positive reviews and has all newcomers acting alongside established Bollywood actor, Gulshan Grover.', ' In 2008, Khan\\'s short film \"Another Kind of Black\" was screened in Cannes Film Festival.']], ['Bhaskar (director)', ['Bhaskar, popularly known as Bommarillu Bhaskar, is a Telugu film director.', ' In his early career, he worked as associate director on films \"Bhadra\" and \"Arya\".', ' His directorial debut \"Bommarillu\" starring Siddarth Narayan and Genelia D\\'souza was a blockbuster, which earned him two Nandi Awards, for best debut director and best original screenplay.', ' His next venture, \"Parugu\" starring Allu Arjun was a box office hit. \"', 'Orange\" starring Ram Charan was his next film, which gained mixed response.', ' His fourth film was \"Ongole Githa\" a revenge drama which was a box office and critical failure.', ' His latest Tamil film Bangalore Naatkal starring Arya, Rana Daggubati, Samantha Ruth Prabhu, Sri Divya, Bobby Simha was released in 2016 and had mixed responses.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.686\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7a2be25542996a35c17118', 'answer': 'Del Amitri', 'question': 'Which band has had more members, The Operation M.D. or Del Amitri?', 'supporting_facts': [['The Operation M.D.', 0], ['Del Amitri', 0], ['Del Amitri', 2]], 'context': [['Justin Currie', ['Justin Robert Currie (born 11 December 1964) is a Scottish singer and songwriter, best known as a founding member of the band Del Amitri and, along with Iain Harvie, is one of only two members of the group to be present throughout its entire existence.']], [\"The Beggars' Guild\", [\"The Beggars' Guild is an American rock band from the state of Georgia.\", ' Its four members are influenced musically by Americana and roots music.', ' They create music in the style of bands such as Tom Petty, Bruce Springsteen, Counting Crows, Led Zeppelin, Johnny Cash, Rolling Stones, Black Crowes, Pedro The Lion, Rich Mullins, Cracker, and Del Amitri.', ' In 2006, they released their first album, an EP called Breaking Me Down.', ' Breaking Me Down was released on the Favorite Gentlemen imprint.']], ['The Silencers (band)', [\"The Silencers are a Scottish rock band formed in London in 1986 by Jimme O'Neill and Cha Burns, two ex-members of the post-punk outfit Fingerprintz.\", ' Their music is characterised by a melodic blend of pop, folk and traditional Celtic influences.', ' Often compared to Scottish bands with a similar sound like Big Country, Del Amitri and The Proclaimers, The Silencers have distinguished themselves with their eclectic sounds, prolific output and continued career.', ' Their first single, \"Painted Moon,\" was a minor international hit and invited critical comparisons to Simple Minds and U2.', ' In 1987 they released their first album \"A Letter From St. Paul,\" which included \"Painted Moon\" and another minor hit, \"I See Red.\"', ' Buoyed by the huge European hit \"Bulletproof Heart\", the band\\'s third album \"Dance to the Holy Man\" is the band\\'s commercial peak to date.', ' Throughout the 1990s, The Silencers saw a popular taste shift away from their songwriter-based style of music toward grunge and electronic music.']], ['Del Amitri discography', ['The discography of Del Amitri, a Scottish pop rock band formed in 1983, includes six studio albums, one live album, two compilation and 19 singles.', ' Four of their studio albums reached the top 10 in the UK Albums Chart.', ' Their first album, which is a self-title album released in May 1985 did not enter the UK Albums Chart at all, and their final studio album \"Can You Do Me Good?', '\", released in 2002, peaked at number 13.', ' The band\\'s most successful studio album was their third \"Change Everything\", which reached second place in the UK Albums Chart.', ' Also the band\\'s compilation album, \"Hatful of Rain: The Best of Del Amitri\", got to fifth place in the UK Albums Chart.', ' The band broke up in 2002.', ' They played a reunion gig at The Hydro Glasgow on 24 January 2014.', ' A live album, \"Into the Mirror\", recorded on the reunion tour in January and February 2014 was released on 20 October 2014.']], ['Waking Hours', ['Waking Hours is the second studio album by Scottish alternative rock band Del Amitri, released in July 1989.', ' It reached number 6 in the UK Albums Chart and featured one of the band\\'s most famous songs, \"Nothing Ever Happens\", which reached number 11 in the UK Singles Chart.', ' The album\\'s opening track, \"Kiss This Thing Goodbye\", entered the top 40 of the US \"Billboard\" Hot 100 when released as a single for the second time.']], ['Hatful of Rain (album)', ['Hatful of Rain (The Best of Del Amitri) is an album by Del Amitri, released in September, 1998.', ' It is a compilation of their greatest hits.']], ['Del Amitri', ['Del Amitri is a Scottish alternative rock band, formed in Glasgow, Scotland in 1983.', \" The band grew out of Justin Currie's Jordanhill College School band and came together after a teenaged Currie placed an advertisement in the window of a music store asking for people who could play to contact him.\", ' The band was formed with the original line-up of Currie (bass and vocals), Iain Harvie (lead guitar), Bryan Tolland (guitar) and Paul Tyagi (drums).', ' Currie and Harvie were the only members of the band to remain present throughout its history.', ' They were also the main songwriters of the group.']], ['The Operation M.D.', ['The Operation M.D. (formerly The Operation) is a Canadian garage rock band created by Dr. Dynamite (Cone of Sum 41) and Dr. Rocco (Todd Morse of HO) as a side project to their other bands.', ' The band\\'s debut album \"We Have an Emergency\" was released in February 2007 by Aquarius Records.', ' Their second album \"Birds + Bee Stings\" was self-released by the band\\'s own label \"Mouth To Mouth Music\" in June 2010.']], ['Iain Harvie', ['Iain Wallace Harvie (born 19 May 1962 in Glasgow, Scotland) is the guitarist with the Scottish rock band Del Amitri.', \" Along with lead singer and bassist Justin Currie, Harvie is one of only two members to be present throughout Del Amitri's history since its 1982 inception.\", \" He is also the co-writer, with Currie, of many of the group's songs.\"]], ['Del Amitri (album)', ['Del Amitri is the eponymous debut album by the Scottish rock band Del Amitri, released in 1985 by Chrysalis Records.', ' A CD reissue in 2003 included 4 bonus tracks.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.686\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abae3205542996cc5e49edc', 'answer': 'Man Booker Prize for Fiction', 'question': 'What award did the writer of Never Let Me Go novel win in 1989?', 'supporting_facts': [['Never Let Me Go (novel)', 0], ['Never Let Me Go (novel)', 1], ['The Remains of the Day', 0], ['The Remains of the Day', 1]], 'context': [['Countrified (Farmer Boys album)', ['Countrified is the first full-length album of the German heavy metal band Farmer Boys.', \" All of the album's songs make reference to farm life or farm animals.\", ' It also has a cover track of Depeche Mode\\'s \"Never Let Me Down Again\".', \" The album is the band's heaviest album ever recorded and it strongly features elements from thrash metal, groove metal and goth metal.\", ' Music videos for \"\"Farm Sweet Farm\"\" and \"\"Never Let Me Down Again\"\" were directed by Nick Lyon.', ' Countrified sold over 10.000 copies.']], ['Grace After Midnight', ['According to the memoir, Pearson says her life was turned around upon learning in prison that a close friend and drug dealer, whom she called \"Uncle\", was killed.', ' Pearson said of her decision to write her book, \"I just want to let people know where I came from ... to share my story.', ' I know the Lord has blessed me.', ' He gave me the power just to tell this story.\"', ' The book also depicts the difficulty with which Pearson attempts to adjust to life after prison; she is fired from two jobs due to her criminal background, which drives her to return temporarily to drug dealing.', ' During an interview about the book, she said, \"Every time I turned around, they kept firing me, so where else could I turn to?', ' The streets never let you down.', \" That's sad to say, but what (was) I supposed to be, starving?\", ' Go to a shelter?\"']], ['Never Let Me Go (2010 film)', [\"Never Let Me Go is a 2010 British dystopian romantic drama film based on Kazuo Ishiguro's 2005 novel of the same name.\", ' The film was directed by Mark Romanek from a screenplay by Alex Garland.', ' \"Never Let Me Go\" is set in an alternative history and centres on Kathy, Ruth and Tommy portrayed by Carey Mulligan, Keira Knightley and Andrew Garfield respectively, who become entangled in a love triangle.', ' Principal photography began in April 2009 and lasted several weeks.', ' The movie was filmed at various locations, including Andrew Melville Hall.', ' \"Never Let Me Go\" was produced by DNA Films and Film4 on a US$15 million budget.']], ['Jakaranda', ['Jakaranda was a pop group consisting of Kenny Wong, Allison DiNonno, and Jacqueline \"Jackie\" Siebert which performed from 1997 to 1998.', ' In the summer of 1998 Kenny Wong was replaced by Ronnie Davidson.', ' The group was a part of Crave Records, a subsidiary of Sony Music Entertainment (Japan), which ceased operations in 1998.', ' The group had moderate attention upon the release of the Disney remake \"The Parent Trap\", where their single \"Never Let You Go\" was featured.', ' A similar version of \"Never Let You Go\" was made by Dario G, an English dance music group in 1997 titled \"Sunchyme.\"', ' It reached No. 2 in the UK singles chart in 1997.', ' The original theme for both these songs was taken from the 1985 song \"Life in a Northern Town\" by The Dream Academy.']], ['Never Let You Go: Shindemo Hanasanai', ['Never Let You Go: Shindemo Hanasanai\\' (Never Let You Go ～死んでも離さない～\") is the first Japanese single by the South Korean boy band 2AM.', ' It was released in January 11, 2012 in three different editions.']], ['Never Let Me Down (song)', ['\"Never Let Me Down\" is a song recorded by English singer David Bowie, serving as the title track for his 1987 studio album of the same name.', ' It was released as the third and final single from the record in 1987, and served as his last single until 1992\\'s \"Real Cool World\" (although a remix of \"Fame\" was released in 1990).', ' \"Never Let Me Down\" was written by the singer himself and Carlos Alomar, while production was handled by Bowie along with David Richards.']], ['Never Let Me Down', ['Never Let Me Down is the seventeenth studio album by David Bowie, released on 20 April 1987 on the label EMI America.', ' Bowie conceived the album as the foundation for a theatrical world tour, writing and recording most of the songs in Switzerland.', ' He considered the record a return to rock and roll music.', ' Three singles were released from the album, \"Day-In Day-Out\", \"Time Will Crawl\" and \"Never Let Me Down\", which all reached the UK Top 40.']], ['The Remains of the Day', ['The Remains of the Day is a 1989 novel by British writer Kazuo Ishiguro.', ' The work was awarded the Man Booker Prize for Fiction in 1989.', ' A film adaptation of the novel, made in 1993 and starring Anthony Hopkins and Emma Thompson, was nominated for eight Academy Awards.']], ['Never Let Me Go (novel)', ['Never Let Me Go is a 2005 dystopian science fiction novel by Japanese-born British author Kazuo Ishiguro.', ' It was shortlisted for the 2005 Booker Prize (an award Ishiguro had previously won in 1989 for \"The Remains of the Day\"), for the 2006 Arthur C. Clarke Award and for the 2005 National Book Critics Circle Award. \"', 'Time\" magazine named it the best novel of 2005 and included the novel in its \"TIME 100 Best English-language Novels from 1923 to 2005\".', ' It also received an ALA Alex Award in 2006.', ' A film adaptation directed by Mark Romanek was released in 2010; a Japanese television drama aired in 2016.']], ['Never Let Me Go (Johnny Ace song)', ['\"Never Let Me Go\" is a blues ballad song by American R&B/blues singer Johnny Ace, written by Joseph Scott and released in 1954 under Duke Records.', ' The song is featured on the albums \"My Songs\" and \"Memorial\".', ' \"Never Let Me Go\" was one of his eighth consecutive top ten R&B hits in a row, including \"My Song\", \"Cross My Heart,\" \"Please Forgive Me,\" \"The Clock,\" \"Pledging My Love,\" \"Saving My Love for You,\" and \"Anymore\".', ' The song was R&B hit and peaked to No. 9 in October 1954 on \"Billboards\" Rhythm & Blues Records chart.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.687\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adc396155429944faac2479', 'answer': 'no', 'question': 'Are both Stanley Elkin and Anne Lamott non-fiction writers? ', 'supporting_facts': [['Stanley Elkin', 0], ['Stanley Elkin', 1], ['Anne Lamott', 0]], 'context': [['Western Writers of America', ['Western Writers of America, founded 1953, promotes literature, both fiction and non-fiction, pertaining to the American West.', ' Although its founders wrote traditional western fiction, the more than six hundred current members also include historians and other non-fiction writers as well as authors from other genres.']], ['Mrs. Ted Bliss', ['Mrs. Ted Bliss is a 1995 novel by American author Stanley Elkin, published by Hyperion Books.', ' It concerns the last eventful years in the life of an old widow.', ' Elkin won the 1995 National Book Critics Circle Award in the fiction category for this work.']], ['Norwegian Non-Fiction Writers and Translators Association', ['Norwegian Non-Fiction Writers and Translators Association']], ['Literary agent', ['A literary agent (sometimes \"publishing agent\", or \"writer\\'s representative\") is an agent who represents writers and their written works to publishers, theatrical producers, film producers and film studios, and assists in the sale and deal negotiation of the same.', ' Literary agents most often represent novelists, screenwriters and non-fiction writers.', ' They are paid a fixed percentage (usually twenty percent on foreign sales and ten to fifteen percent for domestic sales) of the proceeds of sales they negotiate on behalf of their clients.']], ['Steven Zwicker', ['Steven Nathan Zwicker (born June 4, 1943) is an American literary scholar and the Stanley Elkin Professor in the Humanities in Arts and Sciences at Washington University in St. Louis.']], ['George Mills (novel)', ['George Mills is a 1982 novel by American author Stanley Elkin, published by E. P. Dutton.', ' The novel, set in five parts, tells the family history of succeeding generations of characters named George Mills.', ' The story covers more than 1,000 years from the First Crusade in Europe to the Ottoman Empire to present-day America.', ' Elkin won the 1992 National Book Critics Circle Award in the fiction category for the novel.', ' Elkin mentioned \"George Mills\" as one of his favorite novels.', ' The novel is considered Elkin\\'s \"longest and most complexly organized work\".']], ['Gwyn Avenue–Bridge Street Historic District', ['Gwyn Avenue–Bridge Street Historic District is a national historic district located at Elkin, Surry County, North Carolina.', ' The district encompasses 124 contributing buildings and 1 contributing site in a predominantly residential section of Elkin.', ' They were primarily built between about 1891 and 1955 and include notable examples of Queen Anne, Colonial Revival, and Bungalow / American Craftsman architecture.', ' Notable buildings include the Elkin Presbyterian Church (1937, 1944, 1950, 1955, 1961), First Baptist Church (1955, 1968), Alexander Martin Smith House (1893–1897) designed by George Franklin Barber, the Gwyn-Chatham-Gwyn House (c. 1872, 1911, 1936), Richard Gwyn Smith House (c. 1918), and Mason Lillard House (c. 1910).']], ['Stanley Elkin', ['Stanley Lawrence Elkin (May 11, 1930 – May 31, 1995) was an American novelist, short story writer, and essayist.', ' His extravagant, satirical fiction revolves around American consumerism, popular culture, and male-female relation between each other.']], ['Jack Shoemaker', ['Jack Shoemaker (born 1946) is an American editor and publisher, and current editorial director and vice-president at Counterpoint Press in Berkeley, California.', ' Shoemaker has edited and published books under several imprints, including North Point, Pantheon Books, Shoemaker & Hoard, and Counterpoint.', ' Shoemaker has published books by Guy Davenport, Romulus Linney, Gary Snyder, Wendell Berry, Evan S. Connell, MFK Fisher, James Salter, Gina Berriault, Reynolds Price, W.S. Merwin, Michael Palmer, Donald Hall, Anne Lamott, Kay Boyle, Gary Nabhan, Jane Vandenburgh, Carole Maso, and Robert Aitken.', ' Shoemaker supports author-driven literary publishing ventures and mindfulness and political awareness in publishing.', ' Shoemaker was one of the first American publisher of Thich Nhat Hanh, and a major publisher of Wendell Berry.']], ['Anne Lamott', ['Anne Lamott (born April 10, 1954) is an American novelist and non-fiction writer.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.688\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adf8dea5542995534e8c7e4', 'answer': 'the television sitcom \"Barney Miller\"', 'question': 'Somebody’s Daughter’s star Max Gail was most widely known for his role in what other television series?', 'supporting_facts': [[\"Somebody's Daughter\", 0], ['Max Gail', 1]], 'context': [['Tanvi Hegde', ['Tanvi Hegde is an Indian film and television actress.', ' Her family is from Karnataka.', ' She started her career as a child actress in Hindi films and television series.', ' She started her career at the age of 3 by winning the Rasna Baby contest and did a campaign for the same.', ' She is widely known for her lead role of Frooti in the highly successful children\\'s television serial \"Son Pari\", that aired on Star Plus.', ' She also appeared in some episodes of another successful children\\'s show \"Shaka Laka Boom Boom\", also on Star Plus.', ' Hedge has been a part of more than 150 commercials.']], ['Man Against the Mob', ['Man Against the Mob (also known as \"Trouble in the City of Angels\") is a 1988 NBC television movie directed by Steven Hilliard Stern, starring George Peppard, Kathryn Harrold and Max Gail.', ' \"Man Against the Mob\" is a precursor of the 2013 theatrical feature \"Gangster Squad\", in that it deals with the post-war formation of a special LAPD unit set up to suppress Organized Crime in Los Angeles.', ' It may have been inspired by the success of the 1987 theatrical feature \"The Untouchables\", a period drama which also depicted an elite law enforcement unit pitted against mobsters.', ' This was designed around the actor George Peppard as a tough LA cop in the late 1940s.', ' A 1989 TV-movie followup, \"Man Against the Mob: The Chinatown Murders\" is a sequel that also stars Peppard.', ' The first movie was a pilot of a proposed NBC series entitled \"City of Angels\" but ended up panning out as only the two TV movies before George Peppard died in 1994.']], ['Ron Carey (actor)', ['Ronald Joseph Cicenia (December 11, 1935 – January 16, 2007), known as Ron Carey, was an American film and television actor.', ' The 5 ft actor was best known for playing ambitious NYPD Police Officer Carl Levitt on TV\\'s \"Barney Miller\", in which he was almost always surrounded by male actors (and sometimes female guest stars) who stood at least 4 in taller.', \" The series' stars (Hal Linden, Max Gail, Abe Vigoda, Ron Glass, Steve Landesberg) all stood 6 ft or more.\", ' Carey appeared in the recurring role for the last six of the eight seasons of \"Barney Miller\"\\' s run.', ' He first appeared on the show as a criminal, Angelo Molinari (aka The Mole), in Season 2, Episode 22.']], ['Max Gail', ['Maxwell Trowbridge \"Max\" Gail Jr. (born April 5, 1943) is an American actor who has starred in stage, television, and film roles.', ' He most notably portrayed the role of Detective Stan \"Wojo\" Wojciehowicz on the television sitcom \"Barney Miller\".']], [\"Somebody's Daughter\", [\"Somebody's Daughter is a 1992 television film starring Nicollette Sheridan, Nick Mancuso, Boyd Kestner, Michael Cavanaugh, Max Gail and Richard Lineback.\", ' It was directed by Joseph Sargent and written by Lauren Currier.']], ['Star Trek: The Animated Series', ['Star Trek: The Animated Series (originally known simply as Star Trek but also known as \"The Animated Adventures of Gene Roddenberry\\'s Star Trek\") is a 1973 animated science fiction television series set in the \"Star Trek\" universe following the events of \"\" of the 1960s.', ' The animated series was aired under the name Star Trek, but it has become widely known under this longer name (or abbreviated as \"ST: TAS\" or \"TAS\") to differentiate it from the original live-action \".\"', ' The success of the original live action series in syndication, and fan pressure for a \"Star Trek\" revival, led to \"The Animated Series\" from 1973–1974, as the source of new adventures of the \"Enterprise\" crew, the next being the 1979 live-action feature film \"\".']], ['Mariana Ximenes', ['Mariana Ximenes do Prado Nuzzi (born April 26, 1981) is a Brazilian actress.', ' Her first role was in the telenovela \"Fascinação\" in 1998 where she portrayed the role of Emília Gouveia.', ' In the same year she played the role of Ruth Stern in the film \"Caminho dos Sonhos\".', ' In 2000, she played in \"Uga-Uga\" portraying \"Bionda\".', ' This role rise to prominence as she became widely known in Brazil and internationally.', ' Later in 2001, she offered her services in the Portuguese voice over translation in the Canadian/Chinese animated series \"Braceface\", for the character Sharon Spitz.', ' She later played the protagonist in \"Cobras & Lagartos\" in 2006 and Lara in \"A Favorita\", later in 2008.', ' In 2010, she portrayed her first villainous role in the critically acclaimed telenovela \"Passione\".', ' Since 2010 she has appeared in several telenovelas, television series, films and theatre performances.', ' In 2016, she is slated to star in the successor of \"Totalmente Demais\", \"Haja Coração\", together with Malvino Salvador.']], ['Kinshuk Vaidya', ['Kinshuk Vaidya (born 5 April 1991) is an Indian film and television actor.', ' He is widely known for his lead role of Sanju, a guy with a magical pencil, in the Star Plus\\'s highly successful children\\'s television series \"Shaka Laka Boom Boom\", which made him a household name and earned him critical praise.', ' He also worked alongside Kajol, Ajay Devgn and Rishi Kapoor in the children\\'s film \"Raju Chacha\" as Rahul Rai.', ' Despite much anticipation, the film underperformed at the box-office.', ' Vaidya made his comeback after more than a decade with the televion series \"Ek Rishta Saajhedari Ka\"playing the leading role of Aryan Sethia, the series airs on Sony TV.', ' He also appeared in one of the episodes of the anthological series \"Yeh Hai Aashiqui\"that aired on Bindass.']], ['Our Shining Moment', ['Our Shining Moment is a 1991 television family drama film directed by Mark Tinker and starring Cindy Pickett, Max Gail and Don Ameche.', ' It was intended as a pilot for a series which was never produced.', ' It was broadcast on NBC on June 2, 1991.']], ['Steve Purcell', ['Steven Ross Purcell (born 1961) is an American cartoonist, animator, director and game designer.', ' He is most widely known as the creator of \"Sam & Max\", an independent comic book series about a pair of anthropomorphic animal vigilantes and private investigators, for which Purcell received an Eisner Award in 2007.', ' Since being a comic, the series has grown to incorporate an animated television series and several video games.', ' A graduate of the California College of Arts and Craft, Purcell began his career creating comic strips for the college newsletter.', ' He performed freelance work for Marvel Comics and Fishwrap Productions before publishing his first \"Sam & Max\" comic in 1987.', \" Purcell was hired by LucasArts as an artist and animator in 1988, working on several titles within the company's adventure games era.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.689\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abd0077554299700f9d7954', 'answer': 'United States House of Representatives', 'question': 'Lafe Pence was a member of which lower chamber of the United States congress?', 'supporting_facts': [['Lafe Pence', 0], ['United States House of Representatives', 0]], 'context': [['6th United States Congress', ['The Sixth United States Congress was a meeting of the legislative branch of the United States federal government, consisting of the United States Senate and the United States House of Representatives.', \" It met at Congress Hall in Philadelphia, Pennsylvania and in Washington, D.C. from March 4, 1799, to March 4, 1801, during the last two years of John Adams's presidency.\", ' It was the last Congress of the 18th century and the first to convene in the 19th.', ' The apportionment of seats in House of Representatives was based on the First Census of the United States in 1790.', ' Both chambers had a Federalist majority.', ' This was the last Congress in which the Federalist Party controlled the presidency or either chamber of Congress.']], ['Lafe Pence', ['Lafayette (Lafe) Pence (December 23, 1857 – October 22, 1923) was a U.S. Representative from Colorado.']], ['Height of Buildings Act of 1910', ['The Height of Buildings Act of 1910 was an Act of Congress passed by the 61st United States Congress on June 1, 1910 to limit the height of buildings in Washington, D.C. The original act was passed on March 1, 1899 when the 55th United States Congress approved the Height of Buildings Act of 1899.', ' The original act restricted the heights of any type of building in the United States capital city of Washington, D.C., to be no higher than 110 ft .', ' In 1910, the 61st United States Congress enacted a new height restriction law limiting building heights to 130 ft , or the width of the right-of-way of the street or avenue on which a building fronts, whichever is shorter.', ' That is the main law presented by this act.']], ['Sanctity of Life Act', ['The Sanctity of Life Act was a bill first introduced in the United States House of Representatives by Rep. Steve Stockman (R-TX) on July 20, 1995, and cosponsored by Rep. Barbara Cubin (R-WY).', ' It was reintroduced with similar text by Rep. Ron Paul (R-TX) in 2005 in the 109th United States Congress, 110th United States Congress, 111th United States Congress, and the 112th United States Congress.', ' The repeatedly introduced bill sparked advocacy from pro-life activists and opposition from pro-choice activists.', ' The bill has never become law.']], ['United States Senate', ['The United States Senate is the upper chamber of the United States Congress, which along with the United States House of Representatives—the lower chamber—composes the legislature of the United States.']], ['Minnesota Territorial Legislature', ['The Minnesota Territorial Legislature was a bicameral legislative body created by the United States Congress in 1849 as the legislative branch of the government of the Territory of Minnesota.', ' The upper chamber, the Council, and the lower chamber, the House of Representatives, first convened on September 3, 1849.', \" The two chambers served as the territory's legislative body until Minnesota was admitted as a state on May 11, 1858, when the Territorial Legislature was replaced by the Minnesota Legislature.\"]], ['Oregon Territorial Legislature', ['Oregon’s Territorial Legislature was a bicameral legislative body created by the United States Congress in 1848 as the legislative branch of the government of the Oregon Territory.', \" The upper chamber Council and lower chamber House of Representatives first met in July 1849; they served as the region's legislative body until Oregon became a state in February 1859, when they were replaced by the bicameral Oregon State Legislature.\"]], ['Samuel Brenton', ['Samuel Brenton (November 22, 1810 – March 29, 1857) was a U.S. Representative from Indiana; born in Gallatin County, Kentucky.', ' Attended the public schools; was ordained to the Methodist ministry in 1830 and served as a minister; located at Danville, Indiana.', \", in 1834 because of ill health, and studied law; member of the Indiana General Assembly in the Indiana House of Representatives (1838–1841); in 1841, returned to the ministry and served at Crawfordsville, Perryville, Lafayette, and finally at Fort Wayne, where he suffered a paralytic stroke in 1848 and was compelled to abandon his ministerial duties; appointed register of the land office at Fort Wayne on May 2, 1849, and served until July 31, 1851, when he resigned; elected as a Whig to the Thirty-second United States Congress (March 4, 1851 – March 4, 1853); unsuccessful candidate for reelection in 1852 to the Thirty-third United States Congress; elected as an Indiana People's Party candidate to the Thirty-fourth United States Congress; elected as a Republican to the Thirty-fifth United States Congress and served from March 4, 1855, until his death in Fort Wayne, Indiana; interment in Lindenwood Cemetery.\", ' He was replaced by Charles Case in a special election to finish out his term.']], ['United States House of Representatives', ['The United States House of Representatives is the lower chamber of the United States Congress, which along with the Senate composes the legislature of the United States.']], ['Hawaii Senate', ['The Hawaiʻi State Senate is the upper chamber of the Hawaii State Legislature.', ' The senate consists of twenty-five members elected from an equal number of constituent districts across the islands.', ' The senate is led by the President of the Senate, elected from the membership of the body, currently Ron Kouchi.', ' The forerunner of the Hawaii State Senate during the government of the Kingdom of Hawaiʻ i was the House of Nobles originated in 1840.', ' In 1894 the Constitution of the Republic of Hawaii renamed the upper house the present senate.', ' Senators are elected to four-year terms and are not subject to term limits.', ' Like most state legislatures in the United States, the Hawaii State Senate is a part-time body and senators often have active careers outside government.', ' The lower chamber of the legislature is the Hawaiʻi House of Representatives.', ' The membership of the Senate also elects additional officers to include the Senate Vice President, Senate Chief Clerk, Assistant Chief Clerk, Senate Sergeant at Arms and Assistant Sergeant at Arms.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.690\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8205b755429926c1cdadee', 'answer': 'Wright brothers', 'question': \"The Wright Model B was an early pusher biplane designed by what inventors and aviation pioneers who are credited with building the world's first successful airplane?\", 'supporting_facts': [['Wright Model B', 0], ['Wright brothers', 0]], 'context': [['Wright brothers', [\"The Wright brothers, Orville (August 19, 1871 – January 30, 1948) and Wilbur (April 16, 1867 – May 30, 1912), were two American brothers, inventors, and aviation pioneers who are generally credited with inventing, building, and flying the world's first successful airplane.\", ' They made the first controlled, sustained flight of a powered, heavier-than-air aircraft on December 17, 1903, four miles south of Kitty Hawk, North Carolina.', ' In 1904–05 the brothers developed their flying machine into the first practical fixed-wing aircraft.', ' Although not the first to build and fly experimental aircraft, the Wright brothers were the first to invent aircraft controls that made fixed-wing powered flight possible.']], ['Duigan pusher biplane', ['The Duigan pusher biplane (often simply called the Duigan biplane) was an early aircraft which made the first powered flight by an Australian-designed and built machine when it flew in Victoria in 1910.', ' The aircraft was constructed by John Duigan with help from his brother, Reginald, on their family farm at Mia Mia.', ' The effort was especially significant in that the brothers built the aircraft almost entirely by themselves and without input from the pioneering aviation community; a photo-postcard of the Wright Flyer inspired the design and Sir Hiram Maxim\\'s book \"Artificial and Natural Flight\" provided the theoretical basis.']], ['Bristol Boxkite', ['The Boxkite (officially the Bristol Biplane) was the first aircraft produced by the British and Colonial Aeroplane Company (later known as the Bristol Aeroplane Company).', ' A pusher biplane based on the successful Farman III, it was one of the first aircraft types to be built in quantity.', ' As the type was used by Bristol for instruction purposes at their flying schools at Larkhill and Brooklands many early British aviators learned to fly in a Boxkite.', ' Four were purchased in 1911 by the War Office and examples were sold to Russia and Australia.', ' It continued to be used for training purposes until after the outbreak of the First World War.']], ['Albert Berry (parachutist)', ['Captain Albert Berry is one of two people credited as the first person to make a successful parachute jump from a powered airplane.', ' The other contender is Grant Morton, who is reported to have jumped from a Wright Model B flying over Venice Beach, California sometime late in 1911.', \" Morton's pilot was Phil Parmalee.\"]], ['Frank H. Ellis', ['Frank H. Ellis, {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} (October 13, 1893 – July 4, 1979) was an early Canadian aviator and member of the Early Birds of Aviation.', ' He was born in Nottingham, England in 1893 and immigrated to Calgary, Alberta with his family in 1912.', ' With Tom Blakely, he constructed and flew a biplane designed after a Curtiss model in 1914.', ' He was the first Canadian to make a parachute jump from an airplane in Canada, July 5, 1919 at Crystal Beach, Fort Erie, Ontario.', ' Beside his day job as bus driver, Ellis wrote extensively on the history of aviation, and was an avid aircraft model builder.', ' He located several historic Canadian aviation artifacts and arranged for their donation to museums.', ' In 1954 he published \"Canada\\'s Flying Heritage\", the first major study of the History of aviation in Canada.', ' In 1972, he was awarded the Medal of Service of the Order of Canada.', ' He died July 4, 1979 at the age of 85, in North Vancouver, BC.']], ['Sopwith Bat Boat', ['The Sopwith Bat Boats were British flying boats designed and built from 1912 to 1914.', ' A single-engined pusher biplane, the Bat Boat was the first successful flying boat and amphibious aircraft built in the United Kingdom, with examples used by the Royal Navy and by Greece and Germany.']], ['Burgess Model I', ['The Burgess Model I, also known as the \"Burgess I-Scout\" and the \"Coast Defense Hydroaeroplane\", was a United States reconnaissance seaplane built for the Aeronautical Division, U.S. Signal Corps in 1913.', ' It was of conventional Wright Model B design but with an engine mounted amidships in an enclosed fuselage, driving by chains two large pusher propellers mounted on the interplane struts.', ' The undercarriage consisted of twin pontoons.', ' The single example built, S.C. No. 17, was delivered to the Army in January 1913 at the Burgess Company and Curtis factory in Massachusetts, then transported to Florida to complete the training of two officers.', ' After the assignment, it was disassembled and moved to the Philippines in September 1913, where it was in and out of service several times before crashing into the sea near Corregidor on January 12, 1915.', ' It is notable as the first U.S. Army aircraft to conduct two-way radio communication with the ground in December 1914.']], ['Wright Model B', ['The Wright Model B was an early pusher biplane designed by the Wright brothers in the United States in 1910.', ' It was the first of their designs to be built in quantity.', ' Unlike the Model A, it featured a true elevator carried at the tail rather than at the front.', ' It was the last Wright model to have an open-frame tail.', ' The Model B was a dedicated two-seater with the pilot and a passenger sitting side-by-side on the leading edge of the lower wing.']], ['De Schelde Scheldemusch', ['The de Schelde Scheldemusch was a single-seat pusher biplane designed in the Netherlands to be easy and safe to fly.', ' It was one of the first light aircraft to use a tricycle undercarriage.', ' Despite a sales campaign in the UK, only six were built, one being briefly tested by the RAF.', ' A single example of a flying boat version, one of the smallest of this class, was also built.']], ['Allis-Chalmers Model B', ['The Allis-Chalmers model B was a tractor produced by the Allis-Chalmers Manufacturing company from 1937 to 1957.', ' With over 125,000 units produced, the model B became one of the best selling tractors for Allis-Chalmers and most loved tractors of its time.', ' Known best for its versatility and adaptability, the model B was also one of the longest production tractors for Allis-Chalmers as well.', ' Over the years of production the B came in several different variations including the Asparagus B, Potato Special, and the IB industrial tractor.', ' The Model B was designed by Brooks Stevens an industrial designer and graphic designer.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.691\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac31bbd554299741d48a1f9', 'answer': 'Scarface', 'question': 'Vincent Drucci was a member of a rival gang to the mob boss who was known by what nickname?', 'supporting_facts': [['Vincent Drucci', 0], ['Al Capone', 0]], 'context': [['Angelina Veneziano', ['Angelina Veneziano is a fictional character from the American CBS soap opera \"The Young and the Restless\".', ' She is portrayed by Diana DeGarmo, who is famed for being runner-up on the third season of the reality television competition \"American Idol\".', ' She was introduced by former executive producer and head writer Maria Arena Bell on October 31, 2011, as the daughter of mob boss Angelo Veneziano (Mike Starr).', ' Angelina was described as a mob boss daughter and aspiring singer by Zap2it.', ' DeGarmo described her as an over-the-top \"jersey girl\", and noted her flamboyant costumes and voice.', ' The show\\'s executive producer and head writer Maria Arena Bell offered DeGarmo the role of Veneziano after seeing her performance of Penny Pingleton in the musical production of \"Hairspray\".']], ['Richie Fitzpatrick', ['Richard \"Richie\" Fitzpatrick (1880 – November 1, 1904) was a top gunman in the Monk Eastman gang, as well as a former member of the Five Points Gang, during the late 1890s until his death in 1904.', \" He is best known however for the method of eliminating an Eastman rival where he would meet with the person in question and, after being searched, would inform them that he would not follow Eastman's orders instead seeking to defect to the rival gang and as he excused himself to use the bathroom he would retrieve a planted gun and return surprising the person shooting the victim down.\", ' This would later inspire the famous scene in \"The Godfather\" Saga.']], [\"Myles O'Donnell\", [\"Myles O'Donnell was an Irish American bootlegger and mobster during the Roaring Twenties in Chicago during Prohibition.\", \" He was most famous for being the founder of the West-side O'Donnell Mob aka the Westside O'Donnells or West-side gang (no relation to the South Side O'Donnells, a rival gang).\"]], ['Altamira prison brawl', ['The Altamira prison brawl was a deadly fight that occurred on 4 January 2012 in Altamira, Tamaulipas, Mexico.', ' Officials from the state of Tamaulipas confirmed that 31 people were killed, with another thirteen injured.', ' The fight started after a drug gang burst into a section of the prison where they were banned from, attacking their rival gang housed there, triggering the fight.', ' During the altercation, the inmates used several kinds of white arms to kill their opponents.', ' The prisoners also used sticks and knives to massacre the members of the rival gang.']], ['George Musey', ['George Musey, also known as \"one-armed George Musey\", was an associate mob boss in Galveston, Texas, during the 1920s and early 1930s.', ' He, with the \"Beau Brummel of Galveston\" Johnny Jack Nounes, led the Downtown Gang, one of the two gangs which controlled Galveston underworld until the early 1930s.', \" Musey was the gang's top enforcer and would not let anyone ruin the rise of the Downtown Gang.\", ' Bootlegging was his specialty, therefore, when the law would arrive to seize the illegal hooch, Musey always escaped.', ' However, he was convicted on conspiracy liquor charges and sent to Atlanta Penitentiary.', \" He went on to head the gang as Nounes's absence was in effect due to his prison terms.\", ' He was the only right-hand man to gang boss Nounes and he was the best acting boss the gang ever had.', ' He was later assassinated in 1935, eight days after his 35th birthday.']], ['Dan Healy (detective)', ['Daniel F. Healy (c. 1895 – July 8, 1980) was a Chicago detective who became famous when he killed the leader of the North Side Gang, Vincent Drucci, during an altercation, which occurred during the course of an arrest, on April 4, 1927.', ' By 1933 Healy had been made sergeant of the Chicago police']], ['Vincent Drucci', ['Vincent Drucci, also known as \"The Schemer\" (born Vincenzo D\\'Ambrosio; 1898 – April 4, 1927), was an Sicilian-American mobster during Chicago\\'s Prohibition era who was a member of the North Side Gang, Al Capone\\'s best known rivals.', \" A friend of Dean O'Banion, Drucci succeeded him by becoming co-leader.\", ' He is the only US organized crime boss to have been killed by a policeman.']], ['Abraham Weinberg', ['Abraham \"Bo\" Weinberg (January 7, 1900 – September 9, 1935) was a Jewish New York City mobster who became a hitman and chief lieutenant for the Prohibition-era gang boss Dutch Schultz.', ' As Schultz expanded his bootlegging operations into Manhattan during Prohibition, he recruited Abe Weinberg and his brother George into his gang.', ' Abe Weinberg would become one of Schultz\\'s top gunmen during the Manhattan Bootleg Wars and was a later suspect in the high-profile gangland slayings of Jack \"Legs\" Diamond, Vincent \"Mad Dog\" Coll, and mob boss Salvatore Maranzano.']], ['Al Capone', ['Alphonse Gabriel Capone ( ; ] ; January 17, 1899 – January 25, 1947), sometimes known by the nickname Scarface, was an American mobster, crime boss and businessman who attained fame during the Prohibition era as the co-founder and boss of the Chicago Outfit.', ' His seven-year reign as crime boss ended when he was 33 years old.']], ['Yves Buteau', ['Yves Buteau (also known as, Yves \"le Boss\" Buteau) (1951–1983) was known for being a part of motorcycle gangs such as the Hells Angels in Canada, and was murdered by a drug dealer with ties to a rival gang.', ' He began his life of organized crime as a member of the Montreal-based motorcycle gang called, the Popeyes Motorcycle Club.', ' By the mid-1970s, he became president.', ' Buteau would soon play a significant role in establishing the Angels as a major criminal force in Quebec.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.691\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7c54eb554299683c1c62e8', 'answer': 'Heather Langenkamp', 'question': 'The American short horror The sub stars which American actress, make-up artist and producer born in 1964?', 'supporting_facts': [['The Sub', 0], ['The Sub', 1], ['Heather Langenkamp', 0]], 'context': [['The Sub', ['The Sub is an American short horror film directed by Dan Samiljan, co-written by Dave Cain and Samiljan, and produced by Justin Wagman and Noelle Hubbell.', ' The film stars Zoe Jarman, Heather Langenkamp and Brian Stepanek.', ' The film had a successful Kickstarter campaign.']], ['Simon Monjack', ['Simon Mark Monjack (9 March 1970 – 23 May 2010) was a British screenwriter, film director, film producer and make-up artist.', ' He was the husband of American actress Brittany Murphy.']], ['Rick Baker', ['Richard A. \"Rick\" Baker (born December 8, 1950) is an American special make-up effects creator, make-up artist, and special effects supervisor, mostly known for his creature effects; he was also a creature designer.', ' Baker won the Academy Award for Best Makeup and Hairstyling a record seven times from a record eleven nominations, starting in 1981 when he won the inaugural award for \"An American Werewolf in London\".']], ['Monty Westmore', ['Montague George \"Monty\" Westmore (June 12, 1923 – November 13, 2007) was part of the third generation of the Westmore family of American make-up artists in film and television who worked on over 75 films and television series since 1950.', ' He was the brother of make-up artist Michael Westmore and uncle of actress McKenzie Westmore.']], ['William Lemon III', ['William Lemon III (born September 27, 1978) is an American make-up artist, musician and fashion designer known for his special make-up effects used by celebrities including Lady Gaga and Rihanna.']], ['Charis Michelsen', ['Charis Elisa Michelsen (born December 30, 1974) is an American actress, a former model and a make-up artist.', ' Michelsen worked as a model in New York City in her early adulthood before beginning a career as an actress.', ' She appeared in supporting roles in the films \"High Art\" (1998), Martin Scorsese\\'s \"Bringing Out the Dead\" (1999), and \"Wonder Boys\" (2000).']], ['Sutan Amrull', ['Sutan Amrull ( ; born June 14, 1974), also known as Raja and Raja Gemini, is an American make-up artist and drag performer.', ' He is best known for his work on the reality television show \"America\\'s Next Top Model\", serving as the show\\'s make-up artist for nine cycles (fourth through twelve), and as the winner of Season 3 of \"RuPaul\\'s Drag Race\".', \" Amrull's clients include Dita von Teese, Pamela Anderson, Paulina Porizkova, Iman, Tyra Banks, Iggy Azalea, RuPaul, and Twiggy.\", \" Since 2009, Amrull has been make-up artist to singer Adam Lambert for print media, live U.S. appearances, and Lambert's international 2010 Glam Nation Tour.\"]], ['Michael Westmore', ['Michael George Westmore I (born March 22, 1938) is an American make-up artist best known for his work in various \"Star Trek\" productions, winning nine Emmy Awards, and is a member of the Westmore family.', ' He won the Academy Award for Make-up in 1985 for his work on the film \"Mask\".', ' His career began at Universal Studios in 1961, and spanned four decades, including working for the CIA creating make-up kits for spies overseas.']], ['Heather Langenkamp', ['Heather Elizabeth Langenkamp (born July 17, 1964) is an American actress, make-up artist and producer.', ' She is best known for her role as Nancy Thompson in \"A Nightmare on Elm Street\" (1984) and the sequel \"\" (1987), as well as in cult films such as two of Wes Craven\\'s films: \"Shocker\" (1989) and \"Wes Craven\\'s New Nightmare\" (1994).', ' She served as executive producer and narrator to the 2010 documentary \"\".', ' Her other film roles include \"Nickel Mountain\" (1984), \"The Demolitionist\" (1995), \"The Butterfly Room\" (2012), \"Star Trek Into Darkness\" (2013), and \"Truth or Dare\" (2017).', ' Langenkamp is also known for her roles as Marie Lubbock and Amy Boutilier on the sitcom \"Growing Pains\" (1988-1990) and the spin-off series \"Just the Ten of Us\" (1988–1990), and has had numerous television guest appearances.']], ['Farrah Moan', ['Cameron Clayton (born September 11, 1993), better known by his stage name Farrah Moan, is an American drag queen, model, make-up artist and internet personality.', ' He is best known for his participation in the ninth season of Emmy Award-winning reality TV show \"RuPaul\\'s Drag Race\", where he placed eighth.', ' His stage name is a pun on the chemical \"pheromone\", whilst also being a reference to American actress Farrah Fawcett.', ' In some interviews, Clayton jokingly states that his drag surname is a reference to \"being a whore\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.692\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac3d1d9554299204fd21e8a', 'answer': 'four', 'question': 'How many albums had been recorded by Talking Heads by November, 1980?', 'supporting_facts': [['The Name of This Band Is Talking Heads', 0], ['Remain in Light', 0]], 'context': [['Compass Point Studios', ['In the late 1970s and mid-1980s, many musical artists from across the world came to the Bahamas to record music at its facilities.', ' Many producers, including Chris Blackwell himself, used the studio to produce recordings.', ' AC/DC\\'s \"Back In Black\", the second highest selling album ever, was just one of the many albums recorded there.', \" Other well-known artists who recorded there include: Emerson, Lake & Palmer, Julio Iglesias, Serge Gainsbourg, The Rolling Stones, Etta James, Colin James, The Tragically Hip, Grace Jones, Shakira, Celine Dion, U2, Saga, Robert Palmer, Thompson Twins, Tom Tom Club, Talking Heads, Dire Straits, Electric Light Orchestra, Bob Marley, Eric Clapton, James Brown, Iron Maiden, Judas Priest, Roxy Music, Bonnie Tyler, Björk, The B-52's, Lenny Kravitz, Spandau Ballet and David Bowie.\"]], ['Remain in Light', ['Remain in Light is the fourth studio album by American new wave band Talking Heads, released on October 8, 1980, on Sire Records.', \" It was recorded at Compass Point Studios in the Bahamas and Sigma Sound Studios in Philadelphia between July and August 1980 and was produced by the quartet's long-time collaborator Brian Eno.\"]], ['The Name of This Band Is Talking Heads', ['The Name of This Band Is Talking Heads is a double live album by Talking Heads, originally released in 1982.', ' The first disc featured the original quartet in recordings from 1977 and 1979, and the second disc the expanded ten-piece lineup that toured in 1980 and 1981.', ' The album contains live versions of songs that appear on their first four studio albums, \"\", \"More Songs About Buildings and Food\", \"Fear of Music\", and \"Remain in Light.\"', ' The cassette edition of the album included \"Cities\" as a bonus track not included on the vinyl edition – this track has been included on the subsequent CD release.']], ['Talking Heads Africa', ['Talking Heads (Africa) was introduced in Cape Town in 2008 as part of the Infecting the City public art festival.', ' Talking Heads has four core components that form the project.', ' These include: developing a platform for conversation and exchange with and between experts; creating a network of African thought leaders; shooting mini-documentaries that define these leaders and their contributions; developing the tools to make this model work in cities all over the African continent.']], ['Talking Heads (album)', [\"Talking Heads (also known as Brick) is a box set by rock band Talking Heads, containing the band's eight studio albums in DualDisc format with videos and previously unreleased material.\", ' Remixed by Jerry Harrison in Advanced Resolution 5.1 Dolby Surround Sound, \"Brick\" is the first DualDisc release of an artist\\'s entire back catalogue.', ' The albums included in \"Brick\" are:']], ['Tina Weymouth', ['Martina Michèle \"Tina\" Weymouth (born November 22, 1950) is an American musician, best known as a founding member and bassist of the new wave group Talking Heads and its side project Tom Tom Club, which she co-founded with husband and Talking Heads drummer, Chris Frantz.']], ['The Best of Talking Heads', [\"The Best of Talking Heads is a 2004 greatest hits album by Talking Heads, released by Sire/Rhino/Warner Bros., and contains in all 18 tracks, from the beginning to the end of Talking Heads' history.\", ' It was released the same day (August 17, 2004) as the expanded reissue of \"The Name of This Band Is Talking Heads\".', ' The album charted at number 87 on the ARIA Charts and charted at number 96 on the Ultratop Charts in Belgium.']], ['No Talking, Just Head', ['No Talking, Just Head is an album released in 1996 by The Heads, a band composed of Jerry Harrison, Tina Weymouth, and Chris Frantz of Talking Heads, joined by a variety of guest singers.', \" Its name may be seen as an allusion to the fact that Talking Heads' former vocalist, David Byrne, is the only member not involved.\"]], ['Born Under Punches (The Heat Goes On)', ['\"Born Under Punches (The Heat Goes On)\" is the opening track of the Talking Heads 1980 album \"Remain in Light\".', ' The track has a prominent bassline and sets the funk tone of the album.', ' A live rendition of the song was included, with a long bass intro, on the 2004 re-issue of the live album \"The Name of This Band Is Talking Heads\".']], ['Talking Heads: 77', ['Talking Heads: 77 is the debut album by the American rock band Talking Heads, released in September 1977.', ' The single \"Psycho Killer\" reached No. 92 on the Billboard Hot 100 Chart.', ' In 2003, the album was ranked No. 290 on \"Rolling Stone\" magazine\\'s The 500 Greatest Albums of All Time list.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.693\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae12b6755429920d52342e4', 'answer': 'in Singapore', 'question': 'Valuair was acquired in 2005 by a low-cost airline based where?', 'supporting_facts': [['Valuair', 0], ['Valuair', 3], ['Jetstar Asia Airways', 0]], 'context': [['Philippines AirAsia', ['Philippines AirAsia, Inc. (formerly Zest Airways, Inc. dba AirAsia Zest), is a low-cost airline based at Ninoy Aquino International Airport in Metro Manila in the Philippines.', ' The airline is the Philippine affiliate of AirAsia, a low-cost airline based in Malaysia.', ' The airline started as a joint venture between three Filipino investors and AirAsia Investments Ltd., a subsidiary of AirAsia Berhad.']], ['List of low-cost airlines', ['The following is a list of low-cost carriers organized by home country.', ' A low-cost carrier or low-cost airline (also known as a no-frills, discount or budget carrier or airline) is an airline that offers generally low fares in exchange for eliminating many traditional passenger services.', ' Regional airlines, which may compete with low-cost airlines on some routes, are listed at List of regional airlines.']], ['U-Land Airlines', ['U-Land Airlines (Chinese: 瑞聯航空; pinyin: Ruìlián hángkōng) was a Taiwanese low-cost airline.', ' The company was the first low-cost airline company in Asia, which operates domestic and short range international routes.', ' Bankrupted in 2001, it was affiliated to U-Land Building Co., Ltd before it ceases, and is the first airline company bankrupted in Taiwan.']], ['FlyMe', ['Fly Me Europe AB, operating as FlyMe, was a low-cost airline based in Gothenburg, Sweden.', ' It operated flights from Gothenburg, Stockholm and Malmö to destinations within Europe.', ' Its main hub was Göteborg Landvetter Airport, with hubs at Malmö Airport and Stockholm-Arlanda Airport.', ' Beginning in April 2006, it widened its destination network and started daily services to European destinations.', ' Majority owner of the business was English Billionaire John Robert Porter and his Norwegian business partner Christen Ager-Hanssen.', ' They also had a majority interest in another airline Global Supply Systems a British all cargo carrier.', ' FlyMe acquired in late 2006 a 25% stake in Global Supply Systems holding company Riverdon ltd with the aim to create a low cost long haul product.']], ['Valuair', ['Valuair (Chinese: 惠旅航空) was a Singapore-based low-cost carrier.', ' It was launched in 2004, offering initial services to Bangkok and Hong Kong.', ' It differentiates itself from other low-cost carriers in that it offers frills such as a baggage allowance of over 20 kg, in-flight food, allocated seats, and 32 inch seat pitch.', \" Acquired in 2005 by Jetstar Asia Airways, the Valuair brand was retained for Jetstar Asia's scheduled services to major cities in Indonesia until October 2014.\"]], ['Orange Star', ['Orange Star (Chinese: 橘星) is an airline holding company that was formed on July 24, 2005, when Jetstar Asia Airways and Valuair announced their merger in the light of growing competition from other low-cost carriers, price wars and rising fuel costs.', ' Jetstar Asia Chief Executive Officer (CEO) Chong Phit Lian became the CEO of Orange Star.', ' Jetstar Asia employs the Navitaire system, while Valuair uses a Sabre platform.', ' The new company will have a nine-member board, with Qantas CEO Geoff Dixon slated to be chairman.']], ['Jetstar Airways', ['Jetstar Airways Pty Ltd, trading as Jetstar, is an Australian low-cost airline (self-described as \"value based\") headquartered in Melbourne.', ' It is a wholly owned subsidiary of Qantas, created in response to the threat posed by low-cost airline Virgin Blue.', \" Jetstar is part of Qantas' two brand strategy of having Qantas Airways for the premium full-service market and Jetstar for the low-cost market.\", ' Jetstar carries 8.5% of all passengers travelling in and out of Australia.']], ['Jetstar Asia Airways', ['Jetstar Asia Airways Pte Ltd (operating as Jetstar Asia) is a low-cost airline based in Singapore.', \" It is one of the Asian offshoots of parent Jetstar Airways, the low-cost subsidiary airline of Australia's Qantas airline.\", ' It operates services to regional destinations in Southeast Asia to countries such as Myanmar, Cambodia, Malaysia, Philippines, Thailand and Vietnam.', ' It also flies to regional routes in East Asia such as Japan, Taiwan and Hong Kong.', ' It is the main feeder airline for its parent company Jetstar Airways for budget passengers flying to Australia.', ' Its sister airlines include Jetstar in New Zealand, Jetstar Pacific and Jetstar Japan.']], ['Lion Air', ['PT Lion Mentari Airlines, operating as Lion Air, is an Indonesian low-cost airline.', \" Based in Jakarta, Indonesia, Lion Air is the country's largest privately run airline, the second largest low-cost airline in Southeast Asia after AirAsia and the second largest airline of Indonesia, flying to more than 79 destinations in Indonesia, Singapore, Malaysia and Saudi Arabia, as well as charter routes to China, Hong Kong and Macau.\"]], ['Flyglobespan', ['Flyglobespan was a UK low-cost airline based in Edinburgh, UK.', ' It operated scheduled services from five airports across the UK and Ireland to destinations in Europe, North America, North Africa and South Africa.', ' Its main bases were Glasgow International Airport, Edinburgh Airport and Aberdeen Airport.', ' The airline\\'s slogan was \"Award-winning airline\".', ' It went into administration due to financial problems and the airline declared bankruptcy on 16 December 2009.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.693\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adc8977554299438c868de2', 'answer': 'various deities, beings, and heroes', 'question': 'What does the goddess associated with the goddess frigg  consists of what tales?', 'supporting_facts': [['Hlín', 0], ['Norse mythology', 1]], 'context': [['Frigg gas field', ['Frigg gas field is a natural gas field on Norwegian block 25/1 in the North Sea, on the boundary between the United Kingdom and Norway.', ' The field is named after the goddess Frigg.', ' King Olav V of Norway officially opened production on 8 May 1978.', ' Production was closed on 26 October 2004.', ' The field is situated 230 km northwest of Stavanger.', ' Operator for the field was the French oil company Elf Aquitaine, which merged and changed name to Total S.A.']], ['Eir', ['In Norse mythology, Eir (Old Norse \"help, mercy\") is a goddess and/or valkyrie associated with medical skill.', ' Eir is attested in the \"Poetic Edda\", compiled in the 13th century from earlier traditional sources; the \"Prose Edda\", written in the 13th century by Snorri Sturluson; and in skaldic poetry, including a runic inscription from Bergen, Norway from around 1300.', ' Scholars have theorized about whether these three sources refer to the same figure, and debate whether Eir may have been originally a healing goddess and/or a valkyrie.', ' In addition, Eir has been theorized as a form of the goddess Frigg and has been compared to the Greek goddess Hygieia.']], ['Hlín', ['In Norse mythology, Hlín (Old Norse \"protectress\") is a goddess associated with the goddess Frigg.', ' Hlín appears in a poem in the \"Poetic Edda\", compiled in the 13th century from earlier traditional sources, the \"Prose Edda\", written in the 13th century by Snorri Sturluson, and in kennings found in skaldic poetry.', ' Hlín has been theorized as possibly another name for Frigg.']], ['Sága and Sökkvabekkr', ['In Norse mythology, Sága (] , possibly meaning \"seeress\") is a goddess associated with the wisdom Sökkvabekkr (] ; \"sunken bank\", \"sunken bench\", or \"treasure bank\").', ' At Sökkvabekkr, Sága and the god Odin merrily drink as cool waves flow.', ' Both Sága and Sökkvabekkr are attested in the \"Poetic Edda\", compiled in the 13th century from earlier traditional sources, and in the \"Prose Edda\", written in the 13th century by Snorri Sturluson.', ' Scholars have proposed theories about the implications of the goddess and her associated location, including that the location may be connected to the goddess Frigg\\'s fen residence Fensalir and that Sága may be another name for \"Frigg\".']], ['Nanna (Norse deity)', ['In Norse mythology, Nanna Nepsdóttir or simply Nanna is a goddess associated with the god Baldr.', ' Accounts of Nanna vary greatly by source.', ' In the \"Prose Edda\", written in the 13th century by Snorri Sturluson, Nanna is the wife of Baldr and the couple produced a son, the god Forseti.', \" After Baldr's death, Nanna dies of grief.\", \" Nanna is placed on Baldr's ship with his corpse and the two are set aflame and pushed out to sea.\", ' In Hel, Baldr and Nanna are united again.', ' In an attempt to bring back Baldr from the dead, the god Hermóðr rides to Hel and, upon receiving the hope of resurrection from the being Hel, Nanna gives Hermóðr gifts to give to the goddess Frigg (a robe of linen), the goddess Fulla (a finger-ring), and others (unspecified).', ' Nanna is frequently mentioned in the poetry of skalds and a Nanna, who may or may not be the same figure, is mentioned once in the \"Poetic Edda\", compiled in the 13th century from earlier traditional sources.']], ['Gná and Hófvarpnir', ['In Norse mythology, Gná is a goddess who runs errands in other worlds for the goddess Frigg and rides the flying, sea-treading horse Hófvarpnir (Old Norse \"he who throws his hoofs about\", \"hoof-thrower\" or \"hoof kicker\").', ' Gná and Hófvarpnir are attested in the \"Prose Edda\", written in the 13th century by Snorri Sturluson.', ' Scholarly theories have been proposed about Gná as a \"goddess of fullness\" and as potentially cognate to Fama from Roman mythology.', ' Hófvarpnir and the eight-legged steed Sleipnir have been cited examples of transcendent horses in Norse mythology.']], ['Norse mythology', ['Norse mythology is the body of mythology of the North Germanic people stemming from Norse paganism and continuing after the Christianization of Scandinavia and into the Scandinavian folklore of the modern period.', ' The northernmost extension of Germanic mythology, Norse mythology consists of tales of various deities, beings, and heroes derived from numerous sources from both before and after the pagan period, including medieval manuscripts, archaeological representations, and folk tradition.']], ['Fensalir', ['In Norse mythology, Fensalir (Old Norse \"Fen Halls\") is a location where the goddess Frigg dwells.', ' Fensalir is attested in the \"Poetic Edda\", compiled in the 13th century from earlier traditional sources, and the \"Prose Edda\", written in the 13th century by Snorri Sturluson.', \" Scholars have proposed theories about the implications of the location, including that the location may have some connection to religious practices involving springs, bogs, or swamps in Norse paganism, and that it may be connected to the goddess Sága's watery location Sökkvabekkr.\"]], ['Marzanna', ['Marzanna (in Polish), Марена (in Russian), Morė (in Lithuanian), Morana (in Czech, Bulgarian, Slovene, Serbian, Bosnian, and Croatian), or Morena (in Slovak and Macedonian), Maslenitsa (in Russia) and also Mara (in Belarusian and Ukrainian), Maržena, Moréna, Mora or Marmora is a Baltic and Slavic goddess associated with seasonal rites based on the idea of death and rebirth of nature.', \" She is an ancient goddess associated with winter's death and rebirth and dreams.\", ' In Slavic rites the death of the Goddess Marzanna at the end winter, becomes the rebirth of Spring of the Goddess Kostroma (Russian), Lada, Vesna representing the coming of Spring.']], ['Fulla', ['In Germanic mythology, Fulla (Old Norse, possibly \"bountiful\") or Volla (Old High German) is a goddess.', ' In Norse mythology, Fulla is described as wearing a golden band and as tending to the ashen box and the footwear owned by the goddess Frigg, and, in addition, Frigg confides in Fulla her secrets.', ' Fulla is attested in the \"Poetic Edda\", compiled in the 13th century from earlier traditional sources; the \"Prose Edda\", written in the 13th century by Snorri Sturluson; and in skaldic poetry.', ' Volla is attested in the \"Horse Cure\" Merseburg Incantation, recorded anonymously in the 10th century in Old High German, in which she assists in healing the wounded foal of Phol and is referred to as Frigg\\'s sister.', ' Scholars have proposed theories about the implications of the goddess.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.694\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adef5fb5542993a75d263be', 'answer': 'Lambic', 'question': 'Which f beer brewed in the Pajottenland region  American wild ale similar to ', 'supporting_facts': [['American wild ale', 1], ['Lambic', 0]], 'context': [['Lion Red', ['Lion Red is a New Zealand lager-style beer brewed by Lion Breweries, part of the Lion Nathan group.', ' The beer is 4.0% alcohol.', \" Because of its relatively low alcohol content it is widely regarded as an excellent 'session' beer, that is, a beer that can be consumed freely over a long session of time without all the adverse effects of a higher alcohol volume beer.\", ' As such, it is also a favourite of university students, along with similar strength beers such as Speights and DB Draught.']], ['Herkenrode Tripel', ['Herkenrode Tripel is a Belgian Abbey beer brewed for the Herkenrode Foundation by commercial brewery St. Joseph in Opitter (Bree, Flanders in Belgium).', ' It is on the market since July 2009.', ' It has an alcohol by volume percentage of 7%.', ' In June 2009 the Federation of Belgian Brewers awarded the beer the name and the logo of Recognized Belgian Abbey Beer.', ' In the Herkenrode Abbey in Hasselt, which was founded in 1182, beer was brewed until the French Revolution in the brewery of the abbey.']], ['Lambic', ['Lambic is a type of beer brewed in the Pajottenland region of Belgium southwest of Brussels and in Brussels itself at the Cantillon Brewery.', ' Lambic beers include gueuze and kriek lambic.', \" Lambic differs from most other beers in that it is fermented through exposure to wild yeasts and bacteria native to the Zenne valley, as opposed to exposure to carefully cultivated strains of brewer's yeast.\", ' This process gives the beer its distinctive flavour: dry, vinous, and cidery, usually with a sour aftertaste.']], ['Coigneau', ['Coigneau is a Belgian variety of hop which was massively cultivated in the Flemish Aalst-Asse area near Brussels in the nineteenth and first half of the twentieth century.', ' Because of the light bitterness the Coigneau was during a considerable period of time the favorite hop used for Lambic beer brewed in the Pajottenland region of Belgium (southwest of Brussels).']], ['Wild Goose (beer)', ['Wild Goose is a brand of beer brewed by the Logan Shaw Brewing Company of Washington, D.C..', ' The brand is available in traditional English style ale including India Pale Ale and an Oatmeal Stout.']], ['American wild ale', ['American wild ale generally refers to beers brewed in America using yeast or bacteria in addition to \"Saccharomyces cerevisiae\" for fermentation.', ' Such beers may be similar to traditional beers such as Lambic and Oud bruin, and are typically fermented using a strain of \"brettanomyces\" for part or all of the fermentation.', ' The use of brettanomyces can result in a \"funky\" flavor profile.', ' Examples include Jolly Pumpkin Perseguidor, Avery 15 and Brabant, Ommegang Ommegeddon.']], ['Kentucky common beer', ['Kentucky Common Beer is a once-popular style of ale from the area in and around Louisville, Kentucky from the 1850s until Prohibition.', ' This style is rarely brewed commercially today.', ' It was also locally known as \"Dark Cream Common Beer\", \"Cream Beer\" or \"Common Beer\".', \" The beer was top-fermented and wasn't krausened, i.e., it was fermented once and sent out for sale which meant the gravity would be moderate, the carbonation low and the taste full and sweetish.\", ' Like cream ale, it was consumed fresh, usually as draught beer.', ' In 1913 it was estimated that 80% of the beer consumed in Louisville was of this type.', ' Many local breweries made this style of beer exclusively.']], ['Ale', ['Ale is a type of beer brewed using a warm fermentation method, resulting in a sweet, full-bodied and fruity taste.', ' Historically, the term referred to a drink brewed without hops.']], ['Heavy Seas Beer', ['Heavy Seas Beer is brewed by Clipper City Brewing Company, in Baltimore, Maryland.', ' The brewery was established by Hugh Sisson in 1995.', \" Previously, Sisson operated Maryland's first brewpub, Sisson's.\", ' In 2010, the brewery rebranded.', ' While the name of the company remains Clipper City Brewing Company, all of its beer falls under the Heavy Seas brand.', ' Heavy Seas hosts tours on most weekends.', ' It is located at 4615 Hollins Ferry Road, Suite B, in the Halethorpe section of Baltimore.', ' Heavy Seas currently offers a variety of beer styles in approx. 18 states within the United States.', ' Several Heavy Seas beers have been awarded and include the following: Cutlass Amber Lager (a repeat medal winner at the Great American Beer Festival from 2006-2010, bronze medal winner at the 2010 World Beer Cup and silver medal winner at the 2012 World Beer Cup as Heavy Seas Märzen), Powder Monkey Pale Ale (silver medal winner at the 2008 Great American Beer Festival and bronze medal winner at the 2010 World Beer Cup as Heavy Seas Pale Ale), Small Craft Warning Uber Pils (bronze medal winner at the 2004 Great American Beer Festival), Gold Ale (gold medal winner at the 2010 World Beer Cup, bronze medal winner at the 2010 Great American Beer Festival and bronze medal winner at the 2014 Great American Beer Festival as Heavy Seas Gold Ale) and Winter Storm Imperial ESB (gold medal winner at the 2008 World Beer Cup).']], ['Rogue Beard Beer', [\"Rogue Ales Beard Beer is an American wild ale brewed by Rogue Ales of Newport, Oregon using wild yeast originally cultured from nine beard hairs belonging to Rogue Ales' brewmaster, John Maier.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.695\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abbb64455429931dba144b4', 'answer': 'Ford Falcon', 'question': 'The Trans-Tasman Challenge winning car was based off which second generation North american Ford compact car?', 'supporting_facts': [['2016 Touring Car Masters', 4], ['Ford Mustang', 1]], 'context': [['Sport Compact Car', ['Sport Compact Car (or abbreviated as SCC) was an American car magazine that lasted from 1988 to 2009.', ' Sport Compact Car focused on modifying and racing sport compacts, usually import model cars.', ' This publication was known for having a more technical approach than most other typical import car magazines and for the substantial number of project cars they have developed.', ' \"Sport Compact Car\" (SCC) was published monthly by Source Interlink, which acquired it from Primedia in 2007.']], ['Nissan Sylphy', ['The Nissan Sylphy (previously until 2012 known as Nissan Bluebird Sylphy) is a compact car, produced by the Japanese car maker Nissan, as the successor to the Nissan Pulsar.', ' Built since 2000 and currently in its third generation, the second generation is still manufactured for certain markets.', ' Sylphy has also been marketed in export markets under several other model names, including Pulsar and Sentra.']], ['Ford Escort (North America)', ['The North American variant of the Ford Escort is a small family/compact car introduced by Ford in 1980 for the 1981 model year.', ' Adopting the \"Escort\" name used by Ford of Europe since 1968 along with the general design and layout of the third-generation European Escort, it was the successor of the Ford Pinto, which had a tarnished reputation for quality and safety after a widely publicized fuel tank defect.', \" The Escort was one of Ford's most successful models in the 1980s, earning a much better reputation than the Pinto, which faced widely publicized safety issues.\", ' In fact, the Escort was the single best-selling car in its second year in the United States and during most of that decade.']], ['Ford C170 platform', [\"The Ford C170 Platform is Ford's compact car automobile platform from the late 1990s used by the international Ford Focus through its first generation (succeeded by the Ford C1 platform in 2004 outside North America) and continued in use by the North American Ford Focus until 2011 and the Ford Transit Connect until 2013.\"]], ['2016 Touring Car Masters', ['The 2016 Touring Car Masters was an Australian motor racing series for touring cars manufactured between 1 January 1963 and 31 December 1978.', ' It is the tenth running of the Touring Car Masters.', ' John Bowe is the defending series winner.', ' On the September 15, 2016, it was announced that the Touring Cars Masters would combined with the New Zealand Central Muscle Car series for the Bathurst round.', ' The Trans-Tasman Challenge featured over 50 cars from both championships with Glenn Seton winning the round in his Ford Mustang.', ' Dean Perkins was the best placed New Zealander with his Ford Falcon.']], ['Honda Civic', ['The Honda Civic is a line of small cars manufactured by Honda.', ' Originally a subcompact, the Civic has gone through several generational changes, becoming both larger and more upmarket and moving into the compact car segment.', ' EPA guidelines for vehicle size class stipulate a car having combined passenger and cargo room of 110 to is considered a mid-size car, and as such the tenth generation Civic sedan is technically a small-end mid-size car, although it still competes in the compact class.', ' The Civic coupe is still considered a compact car.', ' The Civic currently falls between the Honda Fit and Honda Accord.']], ['Dodge Challenger', ['The Dodge Challenger is the name of four different generations of American automobiles (two of those being pony cars) produced by Dodge in Detroit, Michigan.', ' The Dodge Silver Challenger was produced from 1958 to 1959, as a version of the full-sized Dodge Coronet sedan.', ' From MY 1970 to 1974, the second generation Dodge Challenger pony car was built using the Chrysler E platform, sharing major components with the Plymouth Barracuda.', ' The third generation, from 1978 to 1983, was a badge engineered Mitsubishi Galant Lambda compact car.', ' The fourth, and current generation is a pony car introduced in early 2008 as a rival to the evolved fifth generation Ford Mustang and the fifth generation Chevrolet Camaro.']], ['Ford Mondeo (second generation)', ['The Ford Mondeo MK III (second generation) model was launched by Ford in October 2000.', ' This Mondeo was considerably larger than its predecessor, and although Ford abandoned its New Edge design theme for the second generation, it was their first vehicle to fully benefit from the Prodigy concept car.', ' This gave it an overall effect which many critics felt was more restrained and mature, if much less distinctive.', \" Two of the old car's biggest weaknesses, the modest rear legroom, and uncompetitive diesel version were addressed by a longer wheelbase and the new Duratorq diesel engine.\", \" The basic chassis and suspension design was carried over from the previous generation, which meant that the car continued its predecessor's reputation for class leading handling and ride.\", ' This Mondeo came to Mexico, replacing the North American built Ford Contour, and was sold from 2001 to 2007, when the Ford Fusion replaced it.', ' The North American market Fusion and Ford Five Hundred/Taurus sported very similar styling, inside and out.']], ['Moskvitch 402', ['The Moskvitch 402 is a compact car manufactured by the former Soviet automobile maker MZMA, first time introduced in 1956 as a second generation of the Moskvitch series.', ' In comparison with its predecessor, the Moskvitch-401, the M-402 model featured many improvements which included independent suspension with double wishbones, telescopic shock absorbers, 12-volt electrics, more solid and comfortable car body, more modern trunk, heater, standard car radio, wider viewing range for the driver, etc.']], ['Ford Mustang', ['The Ford Mustang is an American car manufactured by Ford.', ' It was originally based on the platform of the second generation North American Ford Falcon, a compact car.', ' The original 1962 Ford Mustang I two-seater concept car had evolved into the 1963 Mustang II four-seater concept car which Ford used to pretest how the public would take interest in the first production Mustang.', \" The 1963 Mustang II concept car was designed with a variation of the production model's front and rear ends with a roof that was 2.7 inches shorter.\", ' Introduced early on April 17, 1964 (16 days after the Plymouth Barracuda), and thus dubbed as a \"1964½\" by Mustang fans, the 1965 Mustang was the automaker\\'s most successful launch since the Model A.', ' The Mustang has undergone several transformations to its current sixth generation.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.695\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adbfffb5542994650320c39', 'answer': 'September 1901', 'question': 'The McKinley Tariff was framed by the representative who was later assassinated in what month?', 'supporting_facts': [['McKinley Tariff', 0], ['William McKinley', 0]], 'context': [['Sook-ja Kim', ['Sook-ja \"Sue\" Kim (born 1941) is a Korean-born U.S. singer.', ' She started singing with her sisters in a group called The Kim Sisters at the age of 10 and played 13 different instruments.', \" In 1950, the Korean War broke out and Sue Kim's father, Kim Hae-song, who was a famous Korean symphony orchestra conductor was captured by the North Korean army and later assassinated.\", ' Her mother, Lee Nan-young, was left with no home, no food and no money for her and her seven children.', ' Lee was a famous Korean singer who rose to stardom with the song \"Mokpo Tears.\"', \" In 1953, Lee asked the children to perform with her in South Korea's military clubs to support their family.\", ' The GIs donated rock and roll records for the sisters to memorize and sing, and in turn the sisters received chocolate bars and alcohol, which they exchanged for food.', ' The GIs that returned to the States spread the word about the talented trio and in 1958, Tom Ball (who later became The Kim Sisters’ manager) heard about the sisters from one of the returning GIs and went to Korea to recruit them for an Asian act that he was producing.']], ['Wilson–Gorman Tariff Act', ['The Revenue Act or Wilson-Gorman Tariff of 1894 (ch.', ' 349, §73, 28\\xa0Stat.', '\\xa0570 , August 27, 1894) slightly reduced the United States tariff rates from the numbers set in the 1890 McKinley tariff and imposed a 2% income tax.', ' It is named for William L. Wilson, Representative from West Virginia, chair of the U.S. House Ways and Means Committee, and Senator Arthur P. Gorman of Maryland, both Democrats.']], ['Smoot–Hawley Tariff Act', ['The Tariff Act of 1930 (codified at ), otherwise known as the Smoot–Hawley Tariff or Hawley–Smoot Tariff, was an act implementing protectionist trade policies sponsored by Senator Reed Smoot and Representative Willis C. Hawley and signed into law on June 17, 1930.', ' The act raised U.S. tariffs on over 20,000 imported goods.']], ['István Tisza', ['Count István Tisza de Borosjenő et Szeged (archaically English: Stephen Tisza; 22 April 1861 – 31 October 1918) was a Hungarian politician, prime minister, political scientist and member of Hungarian Academy of Sciences.', \" The prominent event in his life was Austria-Hungary's entry into the First World War when he was prime minister for the second time.\", ' He was later assassinated during the Chrysanthemum Revolution on 31 October 1918 - the same day that Hungary terminated its political union with Austria.', ' Tisza supported the dual monarchy of Austria-Hungary and was representative of the then \"liberal-conservative consent\".']], ['McKinley Tariff', ['The Tariff Act of 1890, commonly called the McKinley Tariff, was an act of the United States Congress framed by Representative William McKinley that became law on October 1, 1890.', ' The tariff raised the average duty on imports to almost fifty percent, an act designed to protect domestic industries from foreign competition.', ' Protectionism, a tactic supported by Republicans, was fiercely debated by politicians and condemned by Democrats.', ' The McKinley Tariff was replaced with the Wilson–Gorman Tariff Act in 1894, which promptly lowered tariff rates.']], ['United States elections, 1890', [\"The 1890 United States elections occurred in the middle of Republican President Benjamin Harrison's term.\", ' Members of the 52nd United States Congress were chosen in this election.', ' The Republicans suffered major losses due to the Panic of 1890 and the unpopularity of the McKinley Tariff.', ' The Populist Party also emerged as an important third party.']], ['Revenue Act of 1913', ['The Revenue Act of 1913, also known as the Tariff Act, the Underwood Tariff, the Underwood Act, the Underwood Tariff Act, or the Underwood-Simmons Act (ch.', ' 16, 38\\xa0Stat.', '\\xa0114 , October 3, 1913), re-imposed the federal income tax after the ratification of the Sixteenth Amendment and lowered basic tariff rates from 40% to 25%, well below the Payne-Aldrich Tariff Act of 1909.', ' It was signed into law by President Woodrow Wilson on October 3, 1913 and was sponsored by Alabama Representative Oscar Underwood.']], ['Dingley Act', ['The Dingley Act of 1897 (ch.', ' 11, 30\\xa0Stat.', '\\xa0151 , July 24, 1897), introduced by U.S. Representative Nelson Dingley, Jr., of Maine, raised tariffs in United States to counteract the Wilson–Gorman Tariff Act of 1894, which had lowered rates.', ' Came into effect under William McKinley the first year that he was in office.', ' The McKinley administration wanted to slowly bring back the protectionism that was proposed by the Tariff of 1890.']], ['William McKinley', ['William McKinley (January 29, 1843 – September 14, 1901) was the 25th President of the United States from March 4, 1897 until his assassination in September 1901, six months into his second term.', ' McKinley led the nation to victory in the Spanish–American War, raised protective tariffs to promote American industry, and maintained the nation on the gold standard in a rejection of inflationary proposals.']], ['Thomas Bowman (Iowa politician)', [\"Thomas Bowman (May 25, 1848 – December 1, 1917) was a local official, newspaper publisher, and one-term Democratic U.S. Representative from Iowa's 9th congressional district.\", \" Benefiting from an electoral backlash in 1890 against Republicans for their support of the McKinley Tariff, Bowman's election was a rare nineteenth century Democratic win in traditionally Republican southwestern Iowa.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.697\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adecc7755429975fa854f9d', 'answer': 'Jeremy Hammond', 'question': 'What WikiLeaks using whistleblower is notable for having a hacking organization with a user base of over 1,800,000?', 'supporting_facts': [['HackThisSite', 0], ['HackThisSite', 2], ['Jeremy Hammond', 0], ['Jeremy Hammond', 1]], 'context': [['GameDev.net', ['GameDev.net is a website dedicated to game development, founded by Kevin Hawkins, Dave Astle, and Michael Tanczos among others, in 1999.', ' It serves as a central trade resource and media outlet for the computer and video game industry, with particular regards to hobbyist and independent developers.', ' The site features daily news, articles, forums, job listings, product reviews, book reviews, contests, and regular columns.', ' The site contains many tutorials for hobbyists and professionals alike with a noticeable focus on computer programming.', ' GameDev.net is visited by a variety of people, including both professionals and hobbyists, and has a highly active user base.', \" Previous site wide polls indicate that the site's demographics are predominantly male, with roughly 44.5% of its user base being under 21, and 44.1% of its user base in the 22-30 range.\", ' The site also has a popular forum which boasts advanced topics ranging from artificial intelligence and DirectX programming to the off-topic lounge.']], ['Friends of WikiLeaks', ['Friends of WikiLeaks, sometimes reduced and stylized as FoWL, was a surveillance-resistant social network site created in support of WikiLeaks.', ' Founded in May 2012, the site was intended for those who support WikiLeaks and its activities to perform advocacy.', ' In contrast to more traditional forms of social networking, FoWL aimed at bringing together like-minded people who do not yet know each other.', ' To achieve this goal, the site would ask about the language the user speaks as well as any preferences in the ways of hobbies or other activities.', \" The site would then find six friends who share the user's views within your country, and another six from other parts of the world who speak your language.\", \" If one of those friends cancelled their account or became inactive, a new friend would be matched to the user's circle and would replace the previous inactive one.\"]], ['Ashley Madison data breach', ['In July 2015, a group calling itself \"The Impact Team\" stole the user data of Ashley Madison, a commercial website billed as enabling extramarital affairs.', \" The group copied personal information about the site's user base and threatened to release users' names and personally identifying information if Ashley Madison was not immediately shut down.\", ' On 18 and 20 August, the group leaked more than 25 gigabytes of company data, including user details.']], ['Growth hacking', ['Growth hacking is a process of rapid experimentation across marketing channels and product development to identify the most efficient ways to grow a business.', ' Growth hacking refers to a set of both conventional and unconventional marketing experiments that lead to growth of a business.', ' Growth hackers are marketers, engineers and product managers that specifically focus on building and engaging the user base of a business.', ' Growth hackers often focus on low-cost alternatives to traditional marketing, e.g. using social media, viral marketing or targeted advertising instead of buying advertising through more traditional media such as radio, newspaper, and television.']], ['Sigurdur Thordarson', ['Sigurdur Thordarson (Sigurður Ingi Þórðarson) was born in 1992 in Reykjavík.', ' He is known for his involvement with the whistleblowing organization WikiLeaks, as well as his interactions with the Federal Bureau of Investigation (FBI).', ' In 2010 he was arrested for stealing and leaking classified information about the bank structure in Iceland.', ' He obtained the information from a lawyer that aided wealthy people in tax evasion, whose name was Gunnar Gunnarsson.', ' Gunnar worked for an investment firm called Milestone ehf.', \" Gunnar obtained Sigurdur's services to set up computer systems and delete data.\", ' After Sigurdur was arrested he was introduced to Julian Assange, the editor and founder of WikiLeaks.', \" Sigurdur started his time there in early 2010, and participated in preparing many of WikiLeaks's biggest leaks.\", ' After a year in WikiLeaks service Sigurdur was suspected of embezzling funds from the WikiLeaks online store.', ' WikiLeaks filed criminal charges against Sigurdur with the Metropolitan Police of Iceland, who investigated the case and later dropped it due to lack of evidence.', ' Sigurdur later plead guilty to the embezzlement along with other economic crimes, in 2013.', ' Sigurdur was ordered to pay the victims 15 million ISK (roughly $115,000)']], ['Installed base', ['Installed base (also install base, install[ed] user base or just user base) is a measure of the number of units of a product or service that are actually in use, especially software or an Internet or computing platform, as opposed to market share, which only reflects sales over a particular period.', ' Although the install base number is often created using the number of units that have been sold within a particular period, it is not necessarily restricted to just systems, as it can also be products in general.', ' Because installed base includes machines that may have been in use for many years, it is usually a higher figure than market share .', \" Many people see it as a more reliable indicator of a platform's popularity.\"]], ['Online gaming in China', ['Online gaming in China represents one of the largest and fastest growing Internet business sectors in the world.', ' With 457 million Internet users currently active in the PRC, the country now has the largest online user base in world, of which two-thirds engage in online game play.', ' The average online gamer in China is relatively young (18 to 30 years old), male, and has at least completed a secondary level of education.', ' Demographically the online gaming user base in China is very similar to base of China Internet users, most of whom live in larger cities.']], ['HackThisSite', ['HackThisSite.org, commonly referred to as HTS, is an online hacking and security website founded by Jeremy Hammond, with the site being maintained by a members of the community after his departure.', ' It aims to provide users with a way to learn and practice basic and advanced \"hacking\" skills through a series of challenges in a safe and legal environment.', ' The organization has a user base of over 1,800,000.', ' The actual number of active members is believed to be much lower.', ' The most users online at the same time was 1,995 on February 5, 2012 at 2:46:10 AM CST.']], ['Jeremy Hammond', ['Jeremy Hammond (born January 8, 1985) is a political hacktivist and computer hacker from Chicago.', ' He was convicted and sentenced in November 2013 to 10 years in US Federal Prison for hacking the private intelligence firm Stratfor and releasing the leaks through the whistle-blowing website WikiLeaks.', ' He founded the computer security training website HackThisSite in 2003.']], ['Owen Walker', [\"Owen Thor Walker (online pseudonym AKILL) is a computer hacker living in New Zealand, who was discharged without conviction despite pleading guilty to several charges of 'cybercrime'.\", ' In 2008 he admitted to being the ringleader of an international hacking organization estimated to have caused $26 million worth of damage.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.697\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a801f215542992bc0c4a6b2', 'answer': '2002 Winter Olympics', 'question': 'the first woman to be the chef de mission of an Australian Olympic team won gold medal in which winter Olympics ?', 'supporting_facts': [['Australia at the 2012 Winter Youth Olympics', 1], ['Alisa Camplin', 0]], 'context': [['Australia at the 2012 Winter Youth Olympics', ['Australia competed at the 2012 Winter Youth Olympics in Innsbruck.', ' The chef de mission of the team will be former Olympic champion Alisa Camplin, the first time a woman is the chef de mission of any Australian Olympic team.', ' The Australian team will consist of 13 athletes in 8 sports.']], ['Geoff Henke', ['Geoffrey \"Geoff\" John Henke, AO is a former Australian ice hockey player and Australian Olympic Committee official.', ' He was the chef de mission of the Australian Winter Olympic delegations from 1976 until 1994, and is credited with ending the neglect of winter sports in Australia.']], ['Herman Frazier', ['Herman Ronald \"Herm\" Frazier (born October 29, 1954) is a retired American sprinter.', ' He won gold medals in the 4×400 m relay at the 1976 Olympics and 1975 and 1979 Pan American Games.', ' Individually he earned a bronze medal in the 400 m event at the 1976 Olympics.', ' He served as chef de mission of the 2004 U.S. Olympic team and as the Athletic Director at the University of Alabama at Birmingham and the University of Hawaii.']], ['Canada at the 2011 Pan American Games', ['Canada, which is represented by the Canadian Olympic Committee (COC), has competed at the 2011 Pan American Games in Guadalajara, Mexico from October 14 to 30, 2011.', ' The Canadian team was made up of 492 athletes (256 men and 236 women), the most ever for a non-home Games.', ' On the team all ten provinces and the Northwest Territories are represented.', ' Some sports such as wrestling have sent their strongest team, however sports which offer the most medals (swimming and athletics) have sent for the most part a developmental team.', \" Table Tennis player Anqi Luo (15 years old) was Canada's youngest athlete at the Games while Equestrian athlete Ian Millar (64 years old) was Canada's eldest athlete competing in Guadalajara.\", ' Canada has competed in 35 out of the 36 sports on the program (the exception being basque pelota).', \" On October 4, 2011 women's football star Christine Sinclair was selected to carry the flag during the opening ceremony.\", ' The chef de mission was Jacques Cardyn and the assistant chef de mission was Curt Harnett.']], ['Alisa Camplin', ['Alisa Peta Camplin OAM (born 10 November 1974) is an Australian aerial skier who won gold at the 2002 Winter Olympics, the second ever winter Olympic gold medal for Australia.', ' At the 2006 Winter Olympics, Camplin finished third to receive a bronze medal.', \" She is the first Australian skier to win medals at consecutive Winter Olympics, making her one of Australia's best skiers.\"]], ['Canada at the 2016 Summer Olympics', ['Canada competed at the 2016 Summer Olympics in Rio de Janeiro, Brazil, from August 5 to August 21, 2016.', \" Since the nation's debut in 1900, Canadian athletes had appeared in every edition of the Summer Olympic Games, with the exception of the 1980 Summer Olympics in Moscow because of the United States-led boycott.\", ' The chef de mission was Curt Harnett, appointed in April 2016 after Jean-Luc Brassard, the original chef de mission, resigned his position.']], ['Jana Gantnerová-Šoltýsová', ['Jana Gantnerová-Šoltýsová (born 30 September 1959 in Kežmarok) is a Slovak former alpine skier who competed for Czechoslovakia in the 1976 Winter Olympics, 1980 Winter Olympics, and 1984 Winter Olympics.', 'In December 1980 she won an Alpine Skiing World Cup downhill in Altenmarkt, becoming the first east European skier to win a World Cup race.', ' Her best performance at the Olympics was a fifth place in the downhill in 1984.', ' Since retiring from competition she has served as president of the Slovak Skiing Association, as a member of the Slovak Olympic Committee Executive Board, as a member of the International Ski Federation Alpine Commission, and as Deputy Chef de Mission for the Slovak team at the 2010 Winter Olympics.', ' She is the mother of alpine skier Jana Gantnerová.']], ['Edgar Grospiron', ['Edgar Grospiron (born March 17, 1969) is a French freestyle skier and Olympic champion.', ' He won a gold medal at the 1992 Winter Olympics in Albertville.', ' He received a bronze medal at the 1994 Winter Olympics in Lillehammer.', ' At the 2012 Winter Youth Olympics he was \"Chef de mission\" for the French Team.', ' He was in charge of the Annecy bid for the 2018 Winter Olympics, which did not win.']], ['Sri Lanka at the 2017 Asian Winter Games', ['Sri Lanka is scheduled to compete in the 2017 Asian Winter Games in Sapporo and Obihiro, Japan from February 19 to 26.', \" This marks the country's official debut at the Asian Winter Games and a global Winter sporting event.\", ' The country is scheduled to compete with five athletes in one sport (three disciplines).', ' The team will also consist of two officials and a chef de mission.', ' The chef de mission of the team is Gihan Dalpathdo, the gecretary general of Winter Sport Association of Sri Lanka.']], ['Mike Hay', ['As a player, he had success from 1982 to 1996.', \" He went on to coach the women's team that won gold in Curling at the 2002 Winter Olympics and, partly due to this, was appointed as an MBE in 2004.\", \" He later served as Britain's Olympic performance manager at the 2010 Winter Olympics.\", ' After this he became Chef de Mission for Team GB for the 2014 Winter Olympics.', ' He is brother to David Hay and son of curler Chuck Hay.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.698\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8a06c35542992d82986e57', 'answer': 'Double Crossed', 'question': \"What was the name of Scottish singer-songwriter James Aaron Diamond's first solo album that debuted in 1985?\", 'supporting_facts': [['Double Crossed', 0], ['Jim Diamond (singer)', 0]], 'context': [['Diva (Annie Lennox album)', ['Diva is the debut solo album by the Scottish singer-songwriter Annie Lennox, released in 1992.', ' The album entered the UK album chart at number 1 and has since sold over 1.2 million copies in the UK alone, being certified quadruple platinum.', ' It was also a success in the US, where it was a top 30 hit and has been certified double platinum.', ' \"Diva\" won Album of the Year at the 1993 Brit Awards, and was nominated for Album of the Year at the Grammy Awards the same year.']], ['The Best of Jim Diamond', ['The Best of Jim Diamond is a compilation album of tracks from Scottish singer-songwriter Jim Diamond\\'s first two solo albums \"Double Crossed\" and \"Desire for Freedom\", along with B-sides.']], ['Boys and Girls (album)', ['Boys and Girls is the sixth solo studio album by the English singer and songwriter Bryan Ferry, released in June 1985 by E.G. Records.', \" The album was Ferry's first solo album in seven years and the first since he had disbanded his group Roxy Music in 1983.\", \" The album was Ferry's first and only number one solo album in the UK.\", ' It was certified Platinum by the British Phonographic Industry and contains two UK top 40 hit singles.', \" It is also Ferry's most successful solo album in the US, having been certified Gold for sales in excess of half a million copies there.\"]], ['Double Crossed', [\"Double Crossed was Jim Diamond's solo début album in 1985.\", ' The album featured Diamond\\'s first three solo singles \"I Should Have Known Better\", \"I Sleep Alone at Night\" and \"Remember I Love You\".']], ['City to City', ['City to City is a 1978 album and the second studio album by Scottish singer-songwriter Gerry Rafferty.', \" It was Rafferty's first solo release in six years—and first release of any kind since 1975—due to his tenure in the band Stealers Wheel and subsequent legal proceedings which prevented Rafferty from releasing any new solo recordings for the next three years.\", ' The album was strongly received, peaking at No. 1 in the US and going Platinum, as well as reaching No. 6 in the UK and achieving Gold status. \"', 'Baker Street\", \"Right Down the Line\" and \"Home and Dry\" were successfully released as singles.', ' By October 2010, \"Baker Street\" had reached 5 million plays on British radio.']], ['Necessary Evil (Deborah Harry album)', ['Necessary Evil, is the fifth solo album by the American singer Deborah Harry.', ' Released in September 2007, it is her first solo album in fourteen years.', ' The album contains fourteen tracks (seventeen in some territories), including the first single \"Two Times Blue\", released on the iTunes Store on June 6, 2007.', \" Harry promoted the album on Cyndi Lauper's True Colors Tour 2007, performing a number of songs from the album.\", ' Upon its release in the UK, it debuted at #86.', ' In the US, it debuted at #37 on the Independent Chart.']], ['Kacey Jones', ['Gail Zeiler (April 27, 1950 – September 1, 2016), known professionally as Kacey Jones, was an American singer-songwriter, producer and humorist.', ' After co-writing the Mickey Gilley hit \"I\\'m the One Mama Warned You About\" (credited as Gayle Zeiler), she found success as a performer through the band Ethel & The Shameless Hussies, with whom she released her first album.', ' Later, in 1997, she released her first solo album, \"Men Are Some of My Favorite People\", through Curb Records, before founding her own label, IGO Records, co-founding the Kinkajou Records label with Kinky Friedman and creating two publishing houses—Zamalama Music and Mamalama Music.', ' Since her first solo album, Jones released eight CDs and produced music for both the theatrical comedy \"Nipples to the Wind\" and the movie (and TV series) \"Sordid Lives\".']], ['Songs from the Mirror', ['Songs from the Mirror is the third solo album by Scottish singer-songwriter Fish, released in 1993 as his final album for Polydor.', \" It does not contain any original material; instead it is a cover album featuring Fish's versions of songs by artists who inspired him before his career started.\", ' It reached 46 on the UK Albums Chart.']], ['Wu-Syndicate', ['Wu-Syndicate is a group from Virginia consisting of Joe Mafia, Napoleon, and Myalansky (who named himself after the gangster Meyer Lansky).', ' They were originally called Crime Syndicate but changed their name to Wu-Syndicate when they signed to Wu-Tang Records and became Wu-Tang Clan affiliates.', ' After debuting on the compilation \"\" in 1998, their self-titled debut album \"Wu-Syndicate\" was released in 1999 on both Wu-Tang Records and their own label Slot Time Records.', ' The album was, like most releases from Wu-Tang Clan affiliates during this time enjoyed moderately successful sales with the single \"Where Wuz Heaven\" going gold.', ' Soon after the release there was a dispute of an unknown origin between the group and Wu-Tang Records and the group briefly changed their name to The Syndicate until 2009 with eventual reconciliation and the release of their second official album \"Grimlenz\", produced mostly by Antagonist Dragonspit of Virginia Beach,VA.', ' Both Myalansky and Napoleon continue to work with Joe Mafia but have refused to work with each other since the release of their first album.', ' In an interview Napoleon stated that though they have always clashed, \"Mya is still my dude though regardless\".', ' The group has maintained ties with various members of the Wu-Tang Family.', ' Napoleon is currently working on a project with fellow Wu-Tang alumni Solomon Childs, Shaka Amazulu, and Dexter Wiggle called \"Illuminati Network\".', ' Joe Mafia released his debut solo album \"This One\" in 2002 and founded his own label called 58 West Diamond Street Records.', ' Napoleon released his first solo album, \"Kingpin Wit Da Inkpen\" in 2007 and a mixtape titled \"Mark of the Beast\" in 2011.', ' Myalansky released his first solo album, \"Drastic Measures\" in 2008 and a mixtape a few years later in 2011 \"AMW.Com\".', ' Myalansky has also been working with California rapper Mitchy Slick and has released two more volumes of his \"AMW.Com\" mixtape series.', ' In 2013 Myalansky and Joe Mafia featured on the song \"Golden Age Rapper\" by CHG Unfadable.']], ['Jim Diamond (singer)', ['James Aaron Diamond (28 September 1951 – 8 October 2015) was a Scottish singer-songwriter, best known for his three Top 5 hits.', ' The first was \"I Won\\'t Let You Down\" (1982), as the lead singer in the trio PhD, with Tony Hymas and Simon Phillips.', ' His solo performance, \"I Should Have Known Better\", was a United Kingdom No.1 in 1984.', ' The third track was the theme song from \"Boon\", \"Hi Ho Silver\" which reached No.5 in the UK Singles Chart in 1986.', ' He has also featured as a vocalist on the charity No.1s \"You\\'ll Never Walk Alone\" with The Crowd and \"Let It Be\" with Ferry Aid.', ' His last UK chart success was with \"Young Love (Carry Me Away)\" in 1986.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.699\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac24d725542996366519966', 'answer': 'Menges', 'question': \"Who was Germany's large-scale, low strength fortification unit, disbanded in 1919, named after? \", 'supporting_facts': [['88th Infantry Division (German Empire)', 2], ['Landwehr', 1]], 'context': [['55th Rifle Division (Soviet Union)', ['The 55th Rifle Division that served as a Red Army rifle division during the Great Patriotic War formed for the first time in September, 1925 as a territorial division headquartered at Kursk.', ' When the German invasion began the unit was as Slutsk, but soon came under attack from their armored spearheads and lost most of its strength within days, and was eventually encircled and destroyed at Kiev.', ' A new division was formed along the Volga in December, and was soon sent north to join in the fighting around Demyansk until early 1943.', ' In many respects the 55th was a hard-luck unit; after being destroyed once, it drew assignments to mostly secondary fronts in areas where, due to the terrain and other circumstances, no unit could distinguish itself.', ' By early 1944, the division was reduced to minimal strength for an active formation, and after doing its best in Operation Bagration it was transferred north to the Baltic States and then disbanded to provide replacements for the other units in 61st Army.', ' Elements of the disbanded division were repurposed to other roles in coastal defense and as a naval base garrison, continuing in service until 1956.']], ['Sandwich-structured composite', ['A sandwich-structured composite is a special class of composite materials that is fabricated by attaching two thin but stiff skins to a lightweight but thick core.', ' The core material is normally low strength material, but its higher thickness provides the sandwich composite with high bending stiffness with overall low density.']], ['Pottery fracture', ['Pottery fracture results from stress within a ceramic body due to thermal expansion and contraction, shrinkage, and other forces. Poor drying or uneven compression and alignment of particles can result in low strength.', ' Cracking may appear in greenware as well as each stage of the firing including bisque ware and glazed ware.']], ['38th Battalion (Ottawa), CEF', ['The 38th Battalion, CEF was a unit of the First World War Canadian Expeditionary Force.', ' It was mobilized in Ottawa and recruited in Ottawa, Brockville, Perth, Prescott and Alexandria.', ' An initial draft of five officers and 251 other ranks was sent to England on June 24, 1915.', ' The battalion embarked at Montreal on August 1, 1915, aboard the \"Caledonian\", disembarking in Bermuda on August 12, 1915.', ' Its strength was 35 officers and 959 other ranks.', ' The battalion embarked at Bermuda on May 30, 1916, aboard the \"Grampian\", disembarking in England on June 9, 1916.', ' Its strength was 35 officers and 1001 other ranks.', ' The battalion arrived in France on August 13, 1916, becoming part of the 4th Canadian Division, 12th Canadian Infantry Brigade.', ' It was later reinforced by the 7th Canadian Reserve Battalion.', ' The battalion returned to England on May 6, 1919, arrived in Canada on June 13, 1919, was demobilized in Ottawa on June 15, 1919, and was disbanded by General Order 149 of September 15, 1920.']], ['Landwehr', ['Landwehr, or Landeswehr, is a German language term used in referring to certain national armies, or militias found in nineteenth- and early twentieth-century Europe.', ' In different context it refers to large-scale, low-strength fortifications.', ' In German, the word means \"defence of the country\"; but the term as applied to an insurrectional militia is very ancient, and \"lantveri\" are mentioned in \"Baluzii Capitularia\", as quoted in Hallam\\'s \"Middle Ages\", i. 262, 10th edition.']], ['Controlled low strength material', ['Controlled low strength material, abbreviated CLSM, also known as flowable fill, is a type of weak, runny concrete mix used in construction for non-structural purposes such as backfill or road bases.']], ['Copperhead (climbing)', ['In rock climbing, a copperhead is a small nut with a head made of soft metal on a loop of wire, originally copper or brass, later aluminium.', ' Copperheads are most often placed into small shallow seems and crevices by pounding or hammering them in to place, with a climbing hammer, sometimes with the aid of metal rod, chisel, or punch.', ' The malleability of the soft metal head makes copperheads conform to the rock and grip better than other devices, and are often the only protection that will stay fixed in many placements.', ' Their small size and low strength makes them among the poorest kinds of protection; their main use is in aid climbing where a placement that will just support the weight of the climber can be used to make progress, even though it would be useless in a fall.']], ['88th Infantry Division (German Empire)', ['The 88th Infantry Division (\"88.', ' Infanterie-Division\") was a formation of the Imperial German Army in World War I.', ' The division was formed in November 1914 as the Menges Division (\"Division Menges\"), named after its commander, and made up primarily of Landwehr troops.', ' It became the 88th Infantry Division in August 1915.', ' The division was disbanded in 1919 during the demobilization of the German Army after World War I.']], ['Salt tectonics', ['Salt tectonics is concerned with the geometries and processes associated with the presence of significant thicknesses of evaporites containing rock salt within a stratigraphic sequence of rocks.', ' This is due both to the low density of salt, which does not increase with burial, and its low strength.']], ['Australian Army Training Team Vietnam', ['The Australian Army Training Team Vietnam (AATTV) was a specialist unit of military advisors of the Australian Army that operated during the Vietnam War.', \" Raised in 1962, the unit was formed solely for service as part of Australia's contribution to the war, providing training and assistance to South Vietnamese forces.\", \" Initially numbering only approximately 30 men, the size of the unit grew several times over the following years as the Australian commitment to South Vietnam gradually grew, with the unit's strength peaking at 227 in November 1970.\", ' Members of the team worked individually or in small groups, operating throughout the country from the far south to the Demilitarized Zone (DMZ) in the north.', ' Later they were concentrated in Phuoc Tuy province as Australian forces prepared to withdraw from Vietnam.', ' It is believed to be the most decorated Australian unit to serve in Vietnam; its members received over 100 decorations, including four Victoria Crosses, during its existence.', ' The unit was withdrawn from Vietnam on 18 December 1972 and was disbanded in Australia on 16 February 1973.', ' A total of 1,009 men served with the unit over a period of ten years, consisting of 998 Australians and 11 New Zealanders.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.699\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7ae38c5542992d025e6721', 'answer': 'Teinosuke Kinugasa', 'question': 'Who was older when they died, Teinosuke Kinugasa or Don O. Newland', 'supporting_facts': [['Teinosuke Kinugasa', 0], ['Don O. Newland', 0]], 'context': [['Minoru Inuzuka', ['Minoru Inuzuka (犬塚 稔 , Inuzuka Minoru , 15 February 1901 – 17 September 2007) was a Japanese film director and screenwriter.', ' Starting out as a screenwriter at Shochiku in 1924, he also participated in the production of Teinosuke Kinugasa\\'s \"A Page of Madness\".', ' When Chōjirō Hayashi (later known as Kazuo Hasegawa) became a jidaigeki star at Shochiku, Inuzuka directed many of his films.', ' After World War II, Inuzuka returned to specializing in screenplays and was known for his scripts for the Zatoichi series.', ' He published his autobiography in 2002, and died in 2007 at the age of 106.', ' When he died, he was called the last surviving director to have directed a silent film in the 1920s.', ' Inuzuka wrote scripts for over 150 films and directed over 50.']], ['Don O. Newland', ['Don Oliver Newland (1896–1951) was an American film director and producer whose career consisted largely of itinerant work.', ' From the 1920s until his death, he traveled to cities throughout the United States making films that employed local citizens as stars and extras.', ' Using a standard script, Newland personalized each film according to its location – \"Belvidere\\'s Hero\", \"Staunton\\'s Hero\", \"Janesville\\'s Hero\", \"Huntingdon\\'s Hero\", \"Tyrone\\'s Hero\", \"Wilmington\\'s Hero\", and so on.']], ['A Page of Madness', ['A Page of Madness (狂った一頁 , Kurutta Ippēji or Kurutta Ichipeiji ) is a silent film by Japanese film director Teinosuke Kinugasa, made in 1926.', ' It was lost for forty-five years until being rediscovered by Kinugasa in his storehouse in 1971.', ' The film is the product of an avant-garde group of artists in Japan known as the Shinkankakuha (or School of New Perceptions) who tried to overcome naturalistic representation.']], ['Tsukigata Hanpeita: Hana no maki; Arashi no maki', ['Tsukigata Hanpeita: Hana no maki; Arashi no maki (月形半平太\\u3000花の巻\\u3000嵐の巻) is a 1956 Japanese film directed by Teinosuke Kinugasa.']], ['1954 Cannes Film Festival', ['The 7th Cannes Film Festival was held from 25 March to 9 April 1954.', ' With Jean Cocteau as President of the Jury, the Grand Prix went to the \"Gate of Hell\" by Teinosuke Kinugasa.', ' The festival opened with \"Le Grand Jeu\" by Robert Siodmak.', ' This was the last festival with a predominantly French Jury.']], ['Dedication of the Great Buddha', ['Dedication of the Great Buddha (大仏開眼 , Daibutsu kaigen ) is a 1952 Japanese film directed by Teinosuke Kinugasa.', ' It was entered into the 1953 Cannes Film Festival.']], [\"A Girl Isn't Allowed to Love\", ['A Girl Isn\\'t Allowed to Love a.k.a. \"The Rose Again\" (薔薇いくたびか , Bara ikutabika ) is a 1955 Japanese film directed by Teinosuke Kinugasa.']], ['Teinosuke Kinugasa', ['Teinosuke Kinugasa (衣笠 貞之助 , Kinugasa Teinosuke ) (1 January 1896 – 26 February 1982) was a Japanese actor and film director.', ' He was born in Kameyama, Mie Prefecture and died in Kyoto.', ' Kinugasa won the 1954 Palme d\\'or at Cannes for \"Jigokumon\" (\"The Gate of Hell\").']], ['The Romance of Yushima', ['The Romance of Yushima (婦系図 湯島の白梅 , Onna Keizu Yushima no Shiraume ) (おんなけいず ゆしまのしらうめ), aka \"The White Plum of Yushima\", is a 1955 black-and-white Japanese film directed by Teinosuke Kinugasa.']], ['Jujiro', ['Jujiro (十字路 , Jūjiro ) , also known as \"Crossroads\", \"Crossways\", \"Shadows of the Yoshiwara\" or \"Slums of Tokyo\", is a 1928 silent Japanese film drama directed by Teinosuke Kinugasa.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.700\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abf12005542997719eab661', 'answer': 'Umberto II', 'question': 'Louis, Count of Armagnac has a descendant that includes which final king of Italy?', 'supporting_facts': [['Louis, Count of Armagnac', 2], ['Umberto II of Italy', 0]], 'context': [['Pardaillan', ['Pardaillan, the name of an old French family of Armagnac, of which several members distinguished themselves in the service of the kings of France in the 16th and 17th centuries.', \" Antoine Arnaud de Pardaillan, maréchal de camp, served Henry IV in Franche-Comté, Picardy and Savoy, and was created marquis de Montespan in 1612 and marquis d'Antin in 1615 under Louis XIII.\", ' His grandson Louis Henri Pardaillan, marquis de Montespan, was the husband of Françoise-Athénaïs, marquise de Montespan, the mistress of Louis XIV.', ' Louis Antoine de Pardaillan de Gondrin (1665–1736), legitimate son of the famous marquise, became lieutenant-general of the armies of the king in 1702, governor of the Orléanais, director-general of buildings in.', ' 1708, lieutenant-general in Alsace, member of the council of regency, and minister of state.', \" He was created duc d'Antin in 1711.\", \" The last duc d'Antin, Louis, died in 1757.\"]], ['Jean de Lescun', [\"Jean de Lescun d'Armagnac (died 1473?)\", \", known as “the bastard of Armagnac”, was an ally of king Louis XI of France from before the latter's accession to the throne.\"]], ['Umberto II of Italy', ['Umberto II (Italian: \"Umberto Nicola Tommaso Giovanni Maria di Savoia\" ; 15 September 190418 March 1983), was the last King of Italy.', ' He reigned for 34 days, from 9 May 1946 to 12 June 1946, although he had been \"de facto\" head of state since 1944, and was nicknamed the May King (Italian: Re di Maggio ).']], [\"Louis d'Armagnac, Duke of Nemours\", [\"Louis d'Armagnac, Duke of Nemours (1472, Normandy; 28 April 1503, Cerignola, Italy), known for most of his life as the Count of Guise, was the third son of Jacques d'Armagnac, Duke of Nemours and Louise of Anjou.\"]], ['Charles I, Count of Armagnac', [\"Charles d'Armagnac, born 1425, died June 3, 1497 in Castelnau-de-Montmiral at the age of 72 years, was Count of Armagnac and Rodez from 1473 to 1497.\", \" He was the son of John IV, Count of Armagnac and Rodez, and Isabella d'Évreux.\", \" Because of his brother John V's disloyalty to the king of France, Charles was imprisoned for fifteen years.\", ' His brother was a leader of the league of the public weal against Louis XI, ending his life in a skirmish, allowing Charles to inherit the title of Count of Armagnac.']], ['Armagnac (party)', [\"The Armagnac party was prominent in French politics and warfare during the Hundred Years' War.\", \" It was allied with the supporters of Charles, Duke of Orléans against John the Fearless after Charles' father Louis of Orléans was killed at the orders of the Duke of Burgundy in 1407.\", \" The party took its name from Charles' father-in-law, Bernard VII, Count of Armagnac, who guided the young Duke during his teens and provided much of the financing and some of the seasoned Gascon troops that besieged Paris before their defeat at Saint-Cloud.\", \" Later, John the Fearless was sent back to his lands, and Bernard of Armagnac remained in Paris and, some say, in the queen's bed.\", ' He was assassinated in 1419.']], ['Duke of Lodi', ['The title of Conte di Magenta was created on 30 December 1619 for Don Luigi Melzi, of a Milanese patrician family.', \" His descendant Gaspare, eighth Count, married Maria Teresa d'Eril, daughter and heiress of the Marchese de Fuente Sagrada, and their descendants adopted the name Melzi d'Eril.\", \" Francesco Melzi d'Eril, ninth Count, was made Vice-President of the Italian Republic under Napoleon Bonaparte in 1802, and Grand Chancellor of the Napoleonic Kingdom of Italy in 1805.\", ' On 20 December 1807 he was created Duca di Lodi by Napoleon in his capacity as King of Italy.', ' The Duke was childless, and adopted as his heir his nephew Giovanni Francesco.', ' On his death he was succeeded as tenth Count by his brother Luigi.', ' The Emperor of Austria, as King of Lombardy-Venetia after the Congress of Vienna, recognised the comital title of Magenta in 1816, but did not recognise the Napoleonic dukedom of Lodi.', ' However, Giovanni was given the title of Duca Melzi on 5 September 1818.', ' His son Lodovico, twelfth Count of Magenta, reassumed the title of Duca di Lodi in 1859.', ' The titles of Duca di Lodi and Conte di Magenta were recognised for his successors by ministerial decrees of the new Kingdom of united Italy dated 1890, 1913 and 1939.', ' While there are still heirs to these titles, they were suppressed in 1947 along with all other italian noble titles.']], ['Charlotte of Lorraine', ['Charlotte de Lorraine-Armagnac (6 May 1678 – 21 January 1757) was a Princess of Lorraine by birth and daughter of Louis, Count of Armagnac.', ' She was known as \"Mademoiselle d\\'Armagnac\" and died unmarried.']], ['John II, Count of Armagnac', ['John II, the Hunchback, (born 1333, died May 26, 1384), Count of Armagnac, of Fézensac, Rodez (1371–1384) and Count of Charolais (1364–1384), Viscount Lomagne and Auvillars, he was the son of John I, Count of Armagnac, of Fezensac and Rodez, Viscount Lomagne and Auvillars and Beatrix de Clermont, great-granddaughter of Louis IX of France.']], ['Louis, Count of Armagnac', [\"Louis of Lorraine (7 December 1641 – 13 June 1718) was the Count of Armagnac from his father's death in 1666.\", ' The \"Grand Squire of France\", he was a member of a cadet branch of the House of Guise, itself a cadet branch of the sovereign House of Lorraine.', ' His descendants include Albert II, Prince of Monaco, Umberto II of Italy, and Diana Álvares Pereira de Melo, 11th Duchess of Cadaval.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.700\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8f60175542992414482a90', 'answer': '17 October 2006', 'question': 'When was the game, in which Drew Gehling voices Gord, released for PlayStation 2?', 'supporting_facts': [['Drew Gehling', 0], ['Bully (video game)', 1]], 'context': [[\"Tony Hawk's Pro Skater 3\", ['Tony Hawk\\'s Pro Skater 3 is a skateboarding video game in the \"Tony Hawk\\'s\" series.', ' It was developed by Neversoft and published by Activision in 2001 for the PlayStation 2, PlayStation, GameCube and Game Boy Color.', ' In 2002, it was published for the Xbox, Game Boy Advance, Microsoft Windows, Nintendo 64 and Mac OS.', ' It was the first game released for the PlayStation 2 supporting online play, and the last game released on the Nintendo 64 before discontinuation of the console, a year later in 2003 (2002 in Japan).', ' According to Metacritic, \"Pro Skater 3\" and \"Grand Theft Auto III\" hold an average critic score of 97/100, making them the highest-rated PlayStation 2 games of all time.']], ['NHL 2K10', ['NHL 2K10 is an ice hockey simulation developed by Visual Concepts and published by 2K Sports, part of the \"NHL 2K\" series.', ' It was released on September 15, 2009 for Wii, Xbox 360, PlayStation 3, and PlayStation 2.', ' Randy Hahn and Drew Remenda provide commentary as they did for NHL 2K9.', ' NHL 2K10 was the final 2K Sports ice hockey video game released for the PlayStation 3, PlayStation 2 and Xbox 360, as NHL 2K11 was released only for the iPhone & Wii.']], ['Dancing Stage Fusion', ['Dancing Stage Fusion, abbreviated DS Fusion or simply Fusion, is a music video game released by Konami to the European PlayStation and PlayStation 2 gaming audience on 5 November 2004.', ' In April of the following year, \"Dancing Stage Fusion\" was released as an arcade game.', ' Fusion featured new gameplay features such as EyeToy support for the PlayStation 2 release as well as new music from hit pop artists.', ' The arcade version of Fusion was the first arcade machine in Europe since \"Dancing Stage EuroMix 2\" and set a milestone as the first \"Dance Dance Revolution\" arcade machine produced by Konami since \"Dance Dance Revolution Extreme\" in 2002.', ' The arcade release marked a total game engine upgrade from the old PlayStation-based boards to a new system built on top of an off-the-shelf PlayStation 2.', ' This hardware upgrade would be later featured in the global release of \"Dance Dance Revolution SuperNova\" in 2006.']], ['Pro Evolution Soccer 6', ['Pro Evolution Soccer 6 (also known as World Soccer: Winning Eleven 10 and World Soccer: Winning Eleven X for Xbox 360 in Japan and South Korea, Winning Eleven: Pro Evolution Soccer 2007 in the United States) is a video game developed and published by Konami.', ' Released in 2006 for the PlayStation 2, Xbox 360, and PC platforms and following on the Nintendo DS and PlayStation Portable afterward, \"Pro Evolution Soccer 6\" is the 6th edition of the \"Pro Evolution Soccer\" series for the PlayStation 2, 2nd for the PlayStation Portable and 4th for PC.', ' It is the first game to debut on the Nintendo DS and the Xbox 360.', ' The Xbox 360 version features improved graphics, but retains gameplay similar to the other console versions.', ' The edit mode has been stripped down for the Xbox 360 release, due to time restrictions.', ' The graphics engine on the PC does not utilise the next-gen 360 engine but will again be a direct conversion of the PlayStation 2 engine.']], ['Bully (video game)', ['Bully, originally released in the PAL region as Canis Canem Edit, is an action-adventure video game developed by Rockstar Vancouver and published by Rockstar Games.', ' It was released on 17 October 2006 for PlayStation 2.', ' A remastered version of the game, subtitled \"Scholarship Edition\", was developed by Mad Doc Software and released on 4 March 2008 for Xbox 360 and Wii, and on 21 October 2008 for Microsoft Windows.', ' Bully was re-released on PlayStation 4 available via digital download from PlayStation Network on 22 March 2016.', ' An updated version of the \"Scholarship Edition\", titled \"Anniversary Edition\", was developed by War Drum Studios and was released for Android and iOS on 8 December 2016.']], ['The King of Fighters 2000', ['The King of Fighters 2000 is a 2000 competitive fighting game produced by SNK for the Neo Geo arcade and home platforms.', ' It is the seventh installment in \"The King of Fighters\" series for the Neo Geo, and marks the final game in the series produced by SNK before the bankruptcy.', ' The game was ported to the Sega Dreamcast (in Japan only) and the Sony PlayStation 2 in 2002 .', ' The PlayStation 2 version of the game was released in North America in a two-in-one bundle with its immediate sequel, \"The King of Fighters 2001\", as the first two games to be published by SNK Playmore USA.', ' The Neo-Geo and Sega Dreamcast versions of the game were also included in \"The King of Fighters NESTS Hen\", a compilation released for the Sony PlayStation 2 in Japan.', ' The PlayStation 2 version was re-released on May 3, 2016 for the PlayStation 4 through the PlayStation Network.', ' The game was later released on the Nintendo Switch through the Nintendo eShop service on August 10, 2017.']], ['High Impact Games', ['High Impact Games is an American video game developer based in Burbank, California, formed in 2003 by former members of Insomniac Games and Naughty Dog.', ' In 2007, the company released \"\" for the PlayStation Portable, with a PlayStation 2 port released the next year, and \"Secret Agent Clank\" in 2008, also for the PlayStation Portable.', ' On November 3, 2009, the company released its third game, \"\", for the PlayStation Portable and PlayStation 2.', ' The game was based on the \"Jak & Daxter\" series made by Naughty Dog.', ' In 2010, High Impact Games was developing a remake of Crash Team Racing for PlayStation 3, Xbox 360 and Wii, but the game was canceled by Activision before the prototype initial.', ' An environmental artist, who has worked on some games, revealed that High Impact Games is working on a new project for the Wii.', ' This game has been revealed to be \"\".']], ['Drew Gehling', ['Andrew Shearer Gehling (born October 16, 1982) is an American stage and screen actor, best known for his role as Dr. Pomatter in the Broadway musical \"Waitress\", Garry Marshall\\'s \"Billy & Ray\", and as the voice of Gord in the 2006 video game Bully from Rockstar Games.']], ['Ty the Tasmanian Tiger 3: Night of the Quinkan', ['Ty the Tasmanian Tiger 3: Night of the Quinkan is the third installment of the video game series, \"Ty the Tasmanian Tiger\".', \" It is the only game in the series to be published by Activision, instead of Electronic Arts which published the game's two predecessors.\", ' It was developed by Krome Studios and was released for PlayStation 2, GameCube, Xbox and Game Boy Advance.', ' It was released in North America on October 11, 2005 for Xbox, October 12, 2005 for PlayStation 2 and GameCube and November 1, 2005 for Game Boy Advance.', ' On February 3, 2006, the game was released in limited quantities in PAL regions only for PlayStation 2 and Xbox.', ' The game is also the last in the series to be physically released.']], ['Killzone (series)', [\"Killzone is a first-person and twin sticks shooter series of video games exclusively for Sony Computer Entertainment's (SCE) video game consoles.\", ' The main series and the PlayStation Portable (PSP) installment were developed by Guerrilla Games, a subsidiary of SCE, and the PlayStation Vita installment was developed by Guerrilla Cambridge in the United Kingdom.', ' \"Killzone\" currently consists of six games spanning over the PlayStation 2, PlayStation Portable, PlayStation 3, PlayStation Vita, and the PlayStation 4.', ' The series began on the PlayStation 2 in November 2004 with \"Killzone\", and continued on the PlayStation Portable in October 2006 with \"\". \"', 'Killzone 2\" was released for the PlayStation 3 in February 2009 , and \"Killzone 3\" was released in February 2011 , also for the PlayStation 3. \"\"', ' was released for the PlayStation Vita in September 2013, followed by \"Killzone Shadow Fall\", a launch title for the PlayStation 4, in November 2013.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.701\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5aba7cfe554299232ef4a2fd', 'answer': 'Carabao Cup', 'question': 'The 2017–18 Wigan Athletic F.C. season will be a year in which the team competes in the league cup known as what for sponsorship reasons?', 'supporting_facts': [['2017–18 Wigan Athletic F.C. season', 1], ['EFL Cup', 0]], 'context': [['2017–18 Wigan Athletic F.C. season', [\"The 2017–18 season is Wigan Athletic's 86th year in existence and their first back in League One, after being relegated the previous season.\", ' Along with competing in the league, the club will also participate in the FA Cup, EFL Cup and EFL Trophy.', ' Paul Cook was appointed as manager on 31 May 2017, signing a 3 year deal, following the departure of much of the previous coaching staff the day before.']], ['2007–08 Wigan Athletic F.C. season', [\"The 2007–08 Wigan Athletic F.C. season was the club's 30th season in the Football League and their third season in the Premier League.\"]], ['EFL Cup', [\"The EFL Cup (referred to historically, and colloquially, as simply the League Cup), currently known as the Carabao Cup for sponsorship reasons, is an annual knockout football competition in men's domestic English football.\", \" Organised by the English Football League (EFL), it is open to any club within the top four levels of the English football league system – 92 clubs in total – comprising the top level Premier League, and the three divisions of the English Football League's own league competition (Championship, League One and League Two).\"]], ['Springfield Park (Wigan)', ['Springfield Park was a multi-purpose stadium in Wigan, Greater Manchester.', ' It was the home ground of Wigan Athletic F.C. until the club moved to the new JJB Stadium (now DW Stadium) after the 1998–99 season.', ' At its largest, the stadium held 40,000.', ' In its 102-year existence the ground only saw 32 years as a Football League venue, 11 years for Wigan Borough F.C. and 21 years for Wigan Athletic FC, before it was demolished to make way for a housing estate in 1999.']], ['Trinidad and Tobago League Cup', [\"The Trinidad and Tobago League Cup, or commonly known as the First Citizens Cup for sponsorship reasons, is the league cup style football competition open for Trinidad and Tobago teams competing in the country's TT Pro League.\", ' Similar to the FA Trophy, it is played on a knockout (single elimination) basis in September and October towards the beginning of each Pro League season.', ' Unlike the FA Trophy, where 36 teams enter each season from the top three tiers of the Trinidad and Tobago football league system and the Secondary Schools Football League, only teams from the TT Pro League compete in the league cup.', ' The knockout tournament was inaugurated in 2000 and is currently sponsored by First Citizens Bank.', ' Although the league cup is one of the three major domestic trophies attainable by Trinidad and Tobago league teams, it is perceived as a lower priority than the league championship and the FA Trophy.', ' The current theme is \"Where Winners Reign\", with TT$110,000 to the winner, TT$20,000 to the runners-up, semifinal winners receive TT$7,000, quaterfinal winners receive TT$5,000 and Play-off round winners receive TT$3,000.']], ['1979–80 Wigan Athletic F.C. season', ['The 1979–80 season was the 42nd season in the history of Wigan Athletic F.C. and their second as a professional club in the Football League.']], ['Nick Powell', ['Nicholas Edward \"Nick\" Powell (born 23 March 1994) is an English professional footballer who plays for Wigan Athletic as a midfielder.', ' He began his career at Crewe Alexandra, initially as a forward, making his debut at the age of 16.', ' After impressing for Crewe Alexandra during the 2011–12 season, including scoring in their 2–0 victory in the League Two play-off Final, he moved to Manchester United in July 2012.', ' Unable to break into the Manchester United first team, however, he went on loan to Wigan Athletic, Leicester City and Hull City before being released in June 2016.', ' He joined Wigan Athletic in July 2016.', ' He has represented England at under-16, under-17, under-18, under-19 and under-21 levels.']], ['2006–07 Wigan Athletic F.C. season', [\"The 2006–07 Wigan Athletic F.C. season was the club's 29th season in the Football League and their second season in the Premier League.\"]], ['1978–79 Wigan Athletic F.C. season', ['The 1978–79 season was the 41st season in the history of Wigan Athletic F.C. and their first as a professional club in the Football League.', ' After finishing 2nd in the Northern Premier League during the previous season, the club was nominated to apply for Football League status, and were elected into the league to replace Southport.', ' After a poor start, the club exceeded expectations in the league, and towards the end of the season, a second consecutive promotion seemed possible.', ' The club eventually fell short, finishing the season in 6th place with a total of 55 points.']], ['List of Wigan Borough F.C. players', ['Wigan Borough F.C. was an English football club based in the town of Wigan.', ' The club was founded in 1920 and joined the Lancashire Combination.', ' In 1921, Borough turned professional when their application was accepted to play in the inaugural season of the newly formed Football League Third Division North.', ' The team played in the Football League for ten seasons, with their most successful season coming in 1928–29, finishing fourth in the league and reaching the third round of the FA Cup.', ' Wigan Borough folded during the 1931–32 season due to financial problems, and League football did not return to the town until Wigan Athletic F.C. were elected into the Football League in 1978.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.702\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae780bf554299540e5a55d0', 'answer': '1951', 'question': 'The bomber who carried out a Provisional Irish Republican Army (IRA) assassination attempt against the top tier of the British government in 1984 was born in what year?', 'supporting_facts': [['Brighton hotel bombing', 0], ['Patrick Magee (Irish republican)', 0]], 'context': [['Omagh bombing', ['The Omagh bombing was a car bombing that took place on 15 August 1998 in Omagh, County Tyrone, Northern Ireland.', \" It was carried out by a group calling themselves the Real Irish Republican Army, a Provisional Irish Republican Army (IRA) splinter group who opposed the IRA's ceasefire and the Good Friday Agreement.\", \" The bombing killed 29 people (including a woman pregnant with twins) as well as injuring some 220 others, a death toll even higher than that of any single incident during what were considered 'the Troubles' (1968-10 April 1998).\", ' Telephoned warnings had been sent about 40 minutes beforehand, but were claimed to be inaccurate and police had inadvertently moved people towards the bomb.']], ['Joe Cahill', ['Joe Cahill (Irish: Seosamh Ó Cathail ; 19 May 1920 – 23 July 2004) was a prominent figure in the Irish Republican movement in Northern Ireland and former chief of staff of the Provisional Irish Republican Army (IRA).', ' He joined a junior-republican movement, Na Fianna Eireann, in 1937 and the following year, joined the Irish Republican Army.', ' In 1969, Cahill was a key figure in the founding of the Provisional Irish Republican Army.', ' During his time in the Provisional IRA, Cahill helped import weapons and raise financial support.', ' He served as the chief of staff in 1972, but was arrested the following year when a ship importing weapons was intercepted.', ' After his release, he continued to serve on the IRA Army Council and lead all financial dealings for Sinn Féin.', ' In the 1990s, the IRA and Sinn Féin began to work on seeking peace.', ' Cahill served on the council that called a cessation on 21 July 1996.', ' Cahill attended several of the talks that finally led to the Good Friday Agreement on 10 April 1998.', ' Shortly after the agreement was made, Cahill resigned as treasurer of Sinn Féin.', ' To honour his service, he was made honorary Sinn Féin Vice-President for life.']], ['Dissident Irish Republican campaign', ['Since the Provisional Irish Republican Army (IRA) called a ceasefire and ended its armed campaign in 1997, breakaway groups opposed to the ceasefire (\"dissident Irish republicans\") have continued a low-level armed campaign against the British security forces in Northern Ireland.', ' The main paramilitaries involved are the Real IRA, Continuity IRA and Óglaigh na hÉireann.', ' They have targeted the British Army and Police Service of Northern Ireland (PSNI, successor of the Royal Ulster Constabulary) in gun and bomb attacks, as well as with mortars and rockets.', ' They have also carried out bombings that are meant to cause disruption.', \" However, their campaign has not been as intensive as the Provisional IRA's.\"]], ['Real Irish Republican Army', ['The Real Irish Republican Army or Real IRA (RIRA), is a dissident Irish republican paramilitary organisation which aims to bring about a united Ireland.', \" It formed in 1997 following a split in the Provisional IRA by dissident members, who rejected the IRA's ceasefire that year.\", ' Like the Provisional IRA before it, the RIRA sees itself as the only rightful successor to the original Irish Republican Army and styles itself as \"the Real Irish Republican Army\" in English or \"Óglaigh na hÉireann\" in Irish.', ' It is an illegal organisation in the Republic of Ireland and designated as proscribed terrorist organisation in the United Kingdom and the United States.']], ['Brighton hotel bombing', ['The Brighton hotel bombing was a Provisional Irish Republican Army (IRA) assassination attempt against the top tier of the British government in 1984 that occurred on 12 October 1984 at the Grand Brighton Hotel in Brighton, England.', ' A long-delay time bomb was planted in the hotel by IRA member Patrick Magee, with the purpose of killing Prime Minister Margaret Thatcher and her cabinet, who were staying at the hotel for the Conservative Party conference.', ' Although Thatcher narrowly escaped injury, five people were killed including a sitting Conservative MP, and 31 were injured.']], ['Provisional Irish Republican Army', ['The Provisional Irish Republican Army (Provisional IRA) was an Irish republican paramilitary organization that sought to remove Northern Ireland from the United Kingdom and to bring about an independent socialist republic encompassing all of Ireland.', ' It was the biggest and most active republican paramilitary group during the Troubles.', ' It saw itself as the successor to the original IRA and called itself simply the Irish Republican Army (IRA), or \"Óglaigh na hÉireann\" in Irish, and was widely referred to as such by others.']], ['Patrick Magee (Irish republican)', [\"Patrick Joseph Magee (born 1951) is a former Provisional Irish Republican Army (IRA) Terrorist, best known for planting a bomb in the Brighton's Grand Hotel targeting Prime Minister Margaret Thatcher and her cabinet, which killed two men and three women.\", ' He is sometimes referred to as the \"Brighton bomber\".']], ['Timeline of Real Irish Republican Army actions', ['This is a timeline of actions by the Real Irish Republican Army, also called the Real IRA, an Irish republican paramilitary group.', \" The group was formed in late 1997 by members of the Provisional Irish Republican Army who disagreed with that organisation's ceasefire.\"]], ['Official Irish Republican Army', ['The Official Irish Republican Army or Official IRA (OIRA) was an Irish republican paramilitary group whose goal was to remove Northern Ireland from the United Kingdom and create a \"workers\\' republic\" encompassing all of Ireland.', ' It emerged in December 1969, shortly after the beginning of the Troubles, when the Irish Republican Army split into two factions.', ' The other was the Provisional IRA.', ' Each continued to call itself simply \"the IRA\" and rejected the other\\'s legitimacy.', ' Unlike the \"Provisionals\", the \"Officials\" were Marxist and worked to form a united front with other Irish communist groups, named the Irish National Liberation Front (NLF).', ' The Officials were called the NLF by the Provisionals and were sometimes nicknamed the \"Red IRA\" by others.']], ['The Green Book (IRA)', ['The Green Book is a training and induction manual issued by the Irish Republican Army to new volunteers.', ' It was used by the post-Irish Civil War Irish Republican Army (IRA) and Cumann na mBan, (\"\"League of Women\"\"), along with later incarnations such as the Provisional IRA (PIRA).', ' It includes a statement of military objectives, tactics and conditions for military victory against the British government.', ' This military victory was to be achieved as part of \"\"the ongoing liberation of Ireland from foreign occupiers\"\".', ' The Green Book has acted as a manual of conduct and induction to the organisation since at least the 1950s.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.703\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a713f3c5542994082a3e6eb', 'answer': 'Jacksonville station', 'question': 'Where does the train that runs from NYC and Miami station at in Florida?', 'supporting_facts': [['Jacksonville station', 0], ['Jacksonville station', 1], ['Silver Meteor', 0]], 'context': [['Chapman Field (Miami)', ['Chapman Field (officially the Subtropical Horticulture Research Station) is a horticulture and agronomy research facility of the Agricultural Research Service, a division of the United States Department of Agriculture (USDA), located in Miami, Florida.', ' Dating from 1898, it is one of the oldest entities in South Florida.', ' The USDA also refers to it as the Miami Station.']], ['Silver Meteor', ['The Silver Meteor is a passenger train operated by Amtrak between New York City and Miami, Florida.', ' The first diesel-powered streamliner between New York and Florida, since being introduced by the Seaboard Air Line Railroad (SAL) in 1939, it remains in operation now.', ' The train is part of Amtrak\\'s \"Silver Service\" along with the \"Silver Star\", another former SAL streamliner.']], ['South Miami station', ['South Miami station is a station on the Metrorail rapid transit system in South Miami, Florida.', ' This station is located at the intersection of South Dixie Highway (US 1) and Sunset Drive (SW 72nd Street/SR 986), two blocks west of Red Road (West 57th Avenue).', ' It opened to service May 20, 1984.']], ['Lakeland station', ['Lakeland station is a train station in Lakeland, Florida, that is served by Amtrak, the national passenger rail system of the United States.', ' It is served by the \"Silver Star\" train, which runs daily between New York City and Miami.', ' The station is located on the northern shore of Lake Mirror.']], ['Miami Station, Missouri', ['Miami Station is an unincorporated community in Carroll County, Missouri, United States.', ' Miami Station is located along Missouri Supplemental Route V 2.5 mi northwest of Miami.', ' Miami Station was laid out in 1870 as a station on the St. Louis, Kansas City and Northern Railway; it served as the main freight station for Miami.', ' A post office called Miami Station was established in 1869, and remained in operation until 1951.', ' U.S. Senator William A. Blakley was born in Miami Station.']], ['JMWAVE', ['JMWAVE or JM/WAVE or JM WAVE was the codename for a major secret United States covert operations and intelligence gathering station operated by the CIA from 1961 until 1968.', ' It was headquartered in Building 25 on the South Campus of the University of Miami in Miami, Florida.', ' (This location was formerly the site of Richmond Naval Air Station, an airship base about 12 miles south of the main campus; after the airship base closed, it has been used by the University of Miami since 1948.)', ' The intelligence facility was also referred to as the CIA\\'s \"Miami Station\" or \"Wave Station\".']], ['Jacksonville station', ['Jacksonville station is an Amtrak train station in Jacksonville, Florida, United States.', ' It serves the \"Silver Meteor\" and \"Silver Star\" trains as well as the Thruway Motorcoach to Lakeland.', \" The station lies next door to a freight facility with its own platform and is also just east of Norfolk Southern's Simpson Yard.\"]], ['Miami Worldcenter', ['Miami Worldcenter is a large mixed-use development under construction led by principals Arthur Falcone and Nitin Motwani, spanning several blocks in the Park West neighborhood of Miami, Florida, just north of Downtown.', \" It may include over 25 acres of land, with a convention center, hotel space, residential, as well as copious street level retail and large anchor tenant space, such as Macy's and Bloomingdale's.\", ' The hotel and convention center are planned to be part of the same 55 storey building.', ' The hotel will be very large with 1,800 rooms over the approximately 600000 sqft convention center.', ' One proposed residential building known as the Miami Worldcenter Signature Tower may rise to the maximum 749 ft above sea level permitted in that area.', \" The project may connect with the under construction All Aboard Florida intercity higher-speed rail system's Miami station.\"]], ['Government Center (Miami)', ['Government Center is a district in Downtown Miami, Florida.', ' Bounded roughly by I-95 and NW 3rd Avenue to the west, SW 1st Street to the south, NW 5th Street to the north, and NE 1st Avenue to the east, Government Center is located on the western edge of downtown.', ' The area includes several courthouses, including the historic Miami-Dade County Courthouse and a US district court, the City of Miami police headquarters, city, county, and state offices.', ' The eponymous and most used county transit station, Government Center, serving Metrorail, Metromover, and Metrobus, is located in the bottom of the Stephen P. Clark Government Center building.', ' Directly south of this is the main branch of the Miami-Dade Public Library System, as well as the HistoryMiami museum.', \" Henry Flagler's Florida East Coast Railroad owns roughly nine acres in the middle of Government Center, the site of its former Miami station, which spans several blocks.\", ' While the station was destroyed in 1963 and the site had been used as surface parking lots in the decades following, the railroad never gave up ownership of the property.', ' In mid 2014, the lots were closed down for construction of a new Downtown Miami intercity rail station, as part of their All Aboard Florida system.']], ['Miami station (Amtrak)', ['Miami station is a train station in Miami-Dade County, Florida, on the border of Miami and Hialeah.', ' It is the southern terminus for Amtrak\\'s \"Silver Meteor\" and \"Silver Star\" trains.', ' The station opened in 1978 to replace a 48-year-old Seaboard Air Line Railroad station.', ' It is several blocks away from the Tri-Rail and Metrorail Transfer Station, but there is no direct connection between the stations.', ' The station was scheduled to be replaced by Miami Central Station in Fall 2016, but was delayed to late 2017.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.704\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8297c055429954d2e2eb71', 'answer': 'Cartoon Network Studios', 'question': 'Who does the previous Vice President of Production at the animation studio that is owned and operated by Viacom, currently work for?', 'supporting_facts': [['Brian A. Miller', 0], ['Brian A. Miller', 1], ['Nickelodeon Animation Studio', 0]], 'context': [['Tahani al-Gebali', ['Justice Tahani al-Gebali (Arabic: تهاني الجبالي\\u200e \\u200e , born 9 November 1950-) is the previous Vice President of the Supreme Constitutional Court of Egypt.', ' In 2003 she was appointed by President Hosni Mubarak to her office, becoming by that the first woman to hold a judiciary position in Egypt, and she remained so until 32 Egyptian women were appointed to various judicial positions in 2007.']], ['Nickelodeon Animation Studio', ['Nickelodeon Animation Studio, also known in Burbank as Nickelodeon Studios Burbank, is an American animation studio owned and operated by Viacom through its television network Nickelodeon.', ' The studio produces many of the network\\'s most popular animated series, including \"SpongeBob SquarePants\", \"The Fairly OddParents\", \"Teenage Mutant Ninja Turtles\", \"Harvey Beaks\", \"The Loud House\", \"Bunsen Is a Beast\", and \"Welcome to the Wayne\".', ' It also produces programs for Nicktoons Network, Nick at Nite, TeenNick, and Nick Jr. as well.']], ['Sherry Gunther', ['Sherry Gunther is an American producer known for her work in animation.', ' While at Klasky Csupo, Gunther worked on the television series \"Duckman\", \"Rugrats\", and early seasons of \"The Simpsons\", for which she won a Primetime Emmy Award in 1991.', ' She was made senior vice president of production at Hanna-Barbera Cartoons in 1995.', ' Under Hanna-Barbera President Fred Seibert she oversaw production of Turner Entertainment programs such as \"Dexter\\'s Laboratory\", \"Johnny Bravo\", \"The Real Adventures of Jonny Quest\", and the \"World Premiere Toons\".', \" Sherry then went on to Produce Family Guy and to found Twentieth Television's first in-house Prime-Time animation studio, and produced countless Prime-Time pilots for Imagine Television, Touchstone Television, Twentieth Television, Fox, and Carsey Warner.\", ' She also produced theatrical shorts of Looney Tunes for Warner Bros.', ' Sherry has received four additional Primetime Emmy nominations, Festival Awards, and a Humanitas Award.', ' a Daytime Emmy Award, two CableACE Award nominations, and a Humanitas Prize.']], ['P.A.Works', ['P.A.Works Corporation (株式会社ピーエーワークス , Kabushiki-gaisha Pī Ē Wākusu , short for Progressive Animation Works) is a Japanese animation studio established on November 10, 2000 and is located in Nanto, Toyama, Japan.', \" The company's president and founder Kenji Horikawa once worked for Tatsunoko Production, Production I.G, and Bee Train before forming P.A.Works in 2000.\", ' The main office is located in Toyama, Japan, which is where the drawing and digital photography take place, and production and direction takes place in their Tokyo office.', ' The company is also involved with animation in video games, as well as collaborating in the past with Production I.G and Bee Train to create anime.', ' In January 2008, P.A.Works produced \"True Tears\", their first anime series as the main animation studio involved in the production process.']], ['MTV Tres', ['MTV Tres (stylized as tr3́s, taken from the Spanish word for the number three tres) is an American broadcast, digital cable and satellite television network owned by Viacom Global Entertainment Group, a division of the Viacom Media Networks subsidiary of Viacom.', ' Programming on Tr3s includes lifestyle series, customized music video playlists, news documentaries that celebrate Latino culture, music and artists and English-subtitled programming in Spanish, imported from the MTV España and MTV Latin America channels, as well as Spanish-subtitled programming from MTV.', ' The channel is targeted toward bilingual Latinos and non-Latino Americans aged 12 to 34.', ' The channel is currently headed by executive vice president and general manager Jose Tillan.']], ['Carl Folta', ['Carl Folta is Executive Vice President, Corporate Communications for Viacom.', ' He has served at this post since November 2006.', \" Before that, he served as Executive Vice President, Office of the Chairman, from January 1, 2006, where he served as Sumner Redstone's senior adviser and spokesman.\", ' Previously, he was Executive Vice President, Corporate Relations of the former Viacom Inc., since November 2004.', ' Prior to that, he served as Senior Vice President of Corporate Relations of Viacom from November 1994 to November 2004, and Vice President of Corporate Relations of Viacom from April 1994 to November 1994.', ' Folta held various communications positions at Paramount Communications from 1984 (when the company was known as Gulf+Western, retaining this name until 1989) until joining Viacom through its purchase of Paramount in April 1994.']], ['Brian A. Miller', ['Brian A. Miller is an American television producer and the current Senior Vice President and General Manager of Cartoon Network Studios in Burbank, California, having assumed the title in 2000.', ' He was formerly Vice President of Production at Nickelodeon Animation Studio, Vice President of Production at Hanna-Barbera, and Vice President of Production at DIC Entertainment.', ' He also served as a production supervisor for \"Alvin and the Chipmunks\".', ' He was also the executive in charge of production for various shows in the 1990s and early 2000s such as \"Dexter\\'s Laboratory\", \"CatDog\", \"Hey Arnold!', '\", \"The Angry Beavers\", \"ChalkZone\", \"\", \"The Powerpuff Girls\", \"Captain Planet and the Planeteers\", \"Adventures of Sonic the Hedgehog\", \"Cow and Chicken\", \"Johnny Bravo\", and the first season of \"SpongeBob SquarePants\".']], ['Frank Harmon (executive)', ['Frank Harmon currently serves as executive vice president and chief operating officer for APP Pharmaceuticals.', ' Prior to the spin-off of the proprietary business, Mr Harmon served as executive vice president and chief operating officer of Abraxis Pharmaceutical Products (APP) since September 2006, after having joined Abraxis in May 2006 as the executive vice president of global operations.', ' Mr. Harmon oversees global manufacturing operations as well as the corporate quality assurance and quality control and the supply chain organizations as well as Generic Product Development, Regulatory Affairs and Operational Excellence.', ' Prior to joining Abraxis, Mr. Harmon was the senior vice president, manufacturing operations for the Sterile Technologies Group at Cardinal Health where he was responsible for multiple sites throughout the United States and Puerto Rico.', ' Mr. Harmon has also served as vice president, biopharmaceutical operations for Aventis Behring.']], ['Khumbo Kachali', ['Khumbo Hasting Kachali is a Malawian politician who was Vice President of Malawi from April 2012 to May 2014, serving under President Joyce Banda.', ' He is credited with being the first Vice President from the Northern Region of Malawi.', ' The three previous vice presidents came from the central and southern regions.', ' Kachali previously held a number of cabinet positions between 2004 and 2010.']], ['United States presidential election', ['The election of President and Vice President of the United States is an indirect election in which citizens of the United States who are registered to vote in one of the 50 U.S. states or Washington, D.C. cast ballots for members of the U.S. Electoral College, known as electors.', ' These electors then in turn cast direct votes, known as electoral votes, for President and Vice President.', ' The candidate who receives an absolute majority of electoral votes for President or Vice President (currently, at least 270 out of a total of 538) is then elected to that office.', ' If no candidate receives an absolute majority for President, the House of Representatives chooses the President; if no one receives a majority for Vice President, then the Senate chooses the Vice President.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.704\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a77c15f5542997042120b1c', 'answer': 'Leafcutter John', 'question': 'In the British experimental jazz band Polar Bear, who handles digital devices?', 'supporting_facts': [['Polar Bear (British band)', 0], ['Electronic musical instrument', 0]], 'context': [['Polar Bear (album)', [\"Polar Bear is the eponymous third album by Sebastian Rochford's British jazz band Polar Bear.\"]], ['Electronic musical instrument', ['An electronic musical instrument is a musical instrument that produces sound using electronic circuitry and/or digital devices.', ' Such an instrument sounds by outputting an electrical, electronic or digital audio signal that ultimately is plugged into a power amplifier which drives a loudspeaker, creating the sound heard by the performer and/or listener.']], ['Dim Lit', ['Dim Lit is the debut album by British jazz band Polar Bear, formed and led by drummer Sebastian Rochford.']], ['Shirokuma Cafe', ['Shirokuma Cafe (Japanese: しろくまカフェ , Hepburn: Shirokuma Kafe , lit.', ' \"Polar Bear Café\") is a Japanese manga series by Aloha Higa (ヒガ アロハ , Higa Aroha ) .', ' It revolves around the everyday lives of a group of animals mingling with humans at a café run by a polar bear.', ' An anime adaptation by Studio Pierrot aired in Japan between April 2012 and March 2013.', ' While it never received an official international release (mostly due to its heavy emphasis on Japanese wordplay, which complicates the potential for dubbing into other languages), it is available on the streaming website Crunchyroll as Polar Bear Cafe alongside the television broadcast for global audiences.']], ['Polar Bear (locomotive)', ['Polar Bear is a Bagnall steam locomotive built in 1905 for the Groudle Glen Railway, to supplement the similar but slightly smaller \"Sea Lion\".', ' The two Bagnalls were temporarily taken out of service in the 1920s when they were replaced by a pair of battery locomotives.', ' These proved unsatisfactory, and \"Polar Bear\" and \"Sea Lion\" were returned to traffic.', ' The railway was closed for the duration of World War II, and when the line reopened in the late 1940s only \"Polar Bear\" was returned to traffic.', ' Following the 1962 closure of the GGR, \"Polar Bear\" was sold to the Brockham Museum Trust in 1967.', ' In 1982 it passed, with the rest of the Brockham collection, to the Amberley Museum Railway, where it was returned to traffic in the early 1980s.', ' \"Polar Bear\"\\'s boiler was condemned around 1988, returning to service with a new boiler in 1993.', ' Its boiler certificate expired at the end of 2010; with a retube and work on the firebox being required before a return to service.', ' Since being based at Amberley, \"Polar Bear\" has returned to the Groudle Glen on three occasions (1993, 1996 and 2005) to visit.']], ['Held on the Tips of Fingers', [\"Held On The Tips Of Fingers is the second album by Sebastian Rochford's British jazz band Polar Bear.\"]], ['Polar Bear (British band)', ['Polar Bear is a British experimental jazz band led by drummer Seb Rochford with Pete Wareham and Mark Lockheart on tenor saxophone, Tom Herbert on double bass and Leafcutter John on electronics and occasionally guitar or mandolin.']], ['Same as You', ['Same as You is the sixth studio album by British jazz band Polar Bear.', ' It was released on 30 March 2015 by The Leaf Label.']], ['In Each and Every One', [\"In Each and Every One is the fifth album by Sebastian Rochford's British jazz band Polar Bear.\"]], ['Peepers (album)', [\"Peepers is the fourth album by Sebastian Rochford's British jazz band Polar Bear.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.705\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a83335f5542996488c2e3ec', 'answer': 'Philip Kindred Dick', 'question': 'Which American novelist was notable primarily for publishing science fiction, Allen Drury or Philip K. Dick?', 'supporting_facts': [['Allen Drury', 0], ['Allen Drury', 1], ['Philip K. Dick', 0]], 'context': [['The Father-Thing (collection)', ['The Father-Thing is a collection of science fiction stories by American writer Philip K. Dick.', ' It was first published by Gollancz in 1989 and reprints Volume III of \"The Collected Stories of Philip K. Dick\".', ' It had not previously been published as a stand-alone volume.', ' Many of the stories had originally appeared in the magazines \"If\", \"Science Fiction Adventures\", \"Science Fiction Stories\", \"Orbit\", \"Fantasy and Science Fiction\", \"Imagination\", \"Future\", \"Galaxy Science Fiction\", \"Beyond Fantasy Fiction\", \"Satellite\", \"Science Fiction Quarterly\" and \"Imaginative Tales\".']], ['Allen Drury', ['Allen Stuart Drury (September 2, 1918 – September 2, 1998) was an American novelist.', ' He wrote the 1959 novel \"Advise and Consent\", for which he won the Pulitzer Prize for Fiction in 1960.']], ['The Best of Philip K. Dick', ['The Best of Philip K. Dick is a collection of science fiction stories by Philip K. Dick.', ' It was first published by Del Rey Books in 1977.', ' Many of the stories had originally appeared in the magazines \"Planet Stories\", \"Fantasy and Science Fiction\", \"Space Science Fiction\", \"Imagination\", \"Astounding Stories\", \"Galaxy Science Fiction\", \"Amazing Stories\", \"Science Fiction Stories\" and \"Startling Stories\", as well as the anthologies \"Dangerous Visions\" and \"Star Science Fiction Stories No.3\".']], ['Philip K. Dick Award', ['The Philip K. Dick Award is a science fiction award given annually at Norwescon sponsored by the Philadelphia Science Fiction Society and (since 2005) supported by the Philip K. Dick Trust, and named after science fiction and fantasy writer Philip K. Dick.', \" It has been awarded since 1983, the year after Dick's death.\", ' Works that have received the award are identified on their covers as \"Best Original SF Paperback\".', ' They are awarded to the best original paperback published each year in the US.']], ['Second Variety (1991 collection)', ['Second Variety is a collection of science fiction stories by Philip K. Dick.', ' It was first published by Citadel Twilight in 1991 and reprints Volume III of \"The Collected Stories of Philip K. Dick\" with the addition of the story \"Second Variety\".', ' Many of the stories had originally appeared in the magazines \"If\", \"Science Fiction Adventures\", \"Science Fiction Stories\", \"Orbit\", \"Fantasy and Science Fiction\", \"Imagination\", \"Future\", \"Galaxy Science Fiction\", \"Beyond Fantasy Fiction\", \"Satellite\", \"Science Fiction Quarterly\", \"Imaginative Tales\" and \"Space Science Fiction\".']], ['The Philip K. Dick Reader', ['The Philip K. Dick Reader is a collection of science fiction stories by American writer Philip K. Dick.', ' It was first published by Citadel Twilight in 1997.', ' Many of the stories had originally appeared in the magazines \"If\", \"Science Fiction Adventures\", \"Science Fiction Stories\", \"Orbit\", \"Fantasy and Science Fiction\", \"Imagination\", \"Future\", \"Galaxy Science Fiction\", \"Beyond Fantasy Fiction\", \"Satellite\", \"Imaginative Tales\", \"Fantastic Universe\" and \"Space Science Fiction\".', ' It is identical in content and order to the edition of volume 3 of the Collected Stories of Philip K. Dick produced by the same publisher apart from the substitution of three stories in positions 21-23 of 24 and the omission of the end notes in the Collected Stories edition.', ' At press time, stories 21 and 24 had already been made into successful movie adaptations and stories 22 and 23 had been optioned.']], ['Adjustment Team', ['\"Adjustment Team\" is a science fiction short story by American writer Philip K. Dick.', ' It was first published in \"Orbit Science Fiction\" (September–October 1954, No. 4) with illustration by Faragasso.', ' It was later reprinted in \"The Sands of Mars and Other Stories\" (Australian) in 1958, \"The Book of Philip K. Dick\" in 1973, \"The Turning Wheel and Other Stories\" (United Kingdom) in 1977, \"The Collected Stories of Philip K. Dick\" in 1987 (Underwood–Miller), 1988 (Gollancz, United Kingdom), 1990 (Citadel Twilight, United States), \"Selected Stories of Philip K. Dick\" in 2002 and in \"The Early Work of Philip K. Dick, Volume One: The Variable Man & Other Stories\" in 2009.']], ['Robots, Androids, and Mechanical Oddities', ['Robots, Androids, and Mechanical Oddities: The Science Fiction of Philip K. Dick is a collection of science fiction stories by Philip K. Dick.', ' It was first published by the Southern Illinois University Press in 1984 and was edited by Patricia S. Warrick and Martin H. Greenberg.', ' The stories had originally appeared in the magazines \"Fantasy and Science Fiction\", \"Galaxy Science Fiction\", \"Space Science Fiction\", \"Astounding\", \"Future\", \"Orbit\", \"Science Fiction Stories\", \"Imagination\", \"Amazing Stories\", \"Rolling Stone College Papers\" and \"Playboy\".']], ['Philip K. Dick', ['Philip Kindred Dick (December 16, 1928 – March 2, 1982) was an American writer notable for publishing works of science fiction.']], [\"Allen Drury's University series\", ['Allen Drury\\'s \"University\" series is a trio of novels written by political novelist Allen Drury between 1990 and 1998, which follow a group of university fraternity brothers for a span of over 60 years from 1938 to 2001.', ' Drury graduated from Stanford University in 1939, and his experiences there provided the basis for the series.', ' The novels are set in a different fictional timeline from Drury\\'s 1959 novel \"Advise and Consent\", which earned him a Pulitzer Prize for Fiction.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.706\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab48a7c5542991751b4d7ab', 'answer': 'Aircraft', 'question': 'What is this machine that is able to fly by gaining support from the air whose maneuvering is referenced to a standard rate turn, also known as a rate one turn?', 'supporting_facts': [['Standard rate turn', 0], ['Aircraft', 0]], 'context': [['Mohammad Ibrahim Khan Jhagra', ['Muhammad Ibrahim Khan (Urdu: محمد ابراہیم خان جھگڑا\\u200e ) was a leading politician of the Khyber-Pakhtunkhwa province of Pakistan.', ' Hailing from the outskirts of Peshawar area known as Khalisa, named after the Sikh invaders, he was well known for joining Muhammad Ali Jinnah in the Pakistan movement, and was known by many as \"the king maker of the frontier\".', ' He was popularly known for his resolve and aura and referred to informally by the name Jhagra Khan.', ' Jhagra Khan was a former Congressite and was largely responsible for gaining support for the Pakistan Muslim League (PML) in the Khyber Pakhtunkhwa province for Abdul Qayyum Khan, the first Chief Minister of the province after independence.', ' He was the General Secretary of the PML until he died of throat cancer.', ' He died in England where he was supposed to be treated; his family receiving condolences from many heads of nations including Queen Elizabeth II.', ' He was well known throughout British India, especially in his native province, and was one of the few leaders Jinnah had personally tried, successfully, to recruit in All India Muslim League.']], ['Symbol rate', ['In digital communications, symbol rate, also known as baud rate and modulation rate, is the number of symbol changes, waveform changes, or signaling events, across the transmission medium per time unit using a digitally modulated signal or a line code.', ' The symbol rate is measured in baud (Bd) or symbols per second.', ' In the case of a line code, the symbol rate is the pulse rate in pulses per second.', ' Each symbol can represent or convey one or several bits of data.', ' The symbol rate is related to the gross bitrate expressed in bits per second.']], ['Aircraft', ['An aircraft is a machine that is able to fly by gaining support from the air.', ' It counters the force of gravity by using either static lift or by using the dynamic lift of an airfoil, or in a few cases the downward thrust from jet engines. Common examples of aircraft include airplanes, helicopters, airships (including blimps), gliders, and hot air balloons.']], ['Production leveling', ['Production leveling, also known as production smoothing or – by its Japanese original term – heijunka (平準化 ) , is a technique for reducing the Mura (Unevenness) which in turn reduces muda (waste).', ' It was vital to the development of production efficiency in the Toyota Production System and lean manufacturing.', ' The goal is to produce intermediate goods at a constant rate so that further processing may also be carried out at a constant and predictable rate.']], ['Volumetric flow rate', ['In physics and engineering, in particular fluid dynamics and hydrometry, the volumetric flow rate, (also known as volume flow rate, rate of fluid flow or volume velocity) is the volume of fluid which passes per unit time; usually represented by the symbol Q (sometimes V̇ ).', ' The SI unit is m/s (cubic metres per second).', ' Another unit used is sccm (standard cubic centimeters per minute).']], ['Wire speed', ['In computer networking, wire speed or wirespeed refers to the hypothetical peak physical layer net bitrate (useful information rate) of a cable (consisting of fiber-optical wires or copper wires) combined with a certain digital communication device, interface, or port.', ' For example, the wire speed of Fast Ethernet is 100\\xa0Mbit/s also known as the \"peak bitrate\", \"connection speed\", \"useful bit rate\", \"information rate\", or digital bandwidth capacity.', ' The wire speed is the data transfer rate that a telecommunications standard provides at a reference point between the physical layer and the datalink layer.']], ['Standard rate turn', ['Aircraft maneuvering is referenced to a standard rate turn, also known as a rate one turn (ROT).']], ['LIBOR market model', ['The LIBOR market model, also known as the BGM Model (Brace Gatarek Musiela Model, in reference to the names of some of the inventors) is a financial model of interest rates.', ' It is used for pricing interest rate derivatives, especially exotic derivatives like Bermudan swaptions, ratchet caps and floors, target redemption notes, autocaps, zero coupon swaptions, constant maturity swaps and spread options, among many others.', ' The quantities that are modeled, rather than the short rate or instantaneous forward rates (like in the Heath-Jarrow-Morton framework) are a set of forward rates (also called forward LIBORs), which have the advantage of being directly observable in the market, and whose volatilities are naturally linked to traded contracts.', ' Each forward rate is modeled by a lognormal process under its forward measure, i.e. a Black model leading to a Black formula for interest rate caps.', ' This formula is the market standard to quote cap prices in terms of implied volatilities, hence the term \"market model\".', ' The LIBOR market model may be interpreted as a collection of forward LIBOR dynamics for different forward rates with spanning tenors and maturities, each forward rate being consistent with a Black interest rate caplet formula for its canonical maturity.', ' One can write the different rates dynamics under a common pricing measure, for example the forward measure for a preferred single maturity, and in this case forward rates will not be lognormal under the unique measure in general, leading to the need for numerical methods such as monte carlo simulation or approximations like the frozen drift assumption.']], ['G.711', ['G.711 is an ITU-T standard for audio companding.', ' It is primarily used in telephony.', ' The standard was released for usage in 1972.', ' Its formal name is \"Pulse code modulation (PCM) of voice frequencies\".', ' It is a required standard in many technologies, for example in H.320 and H.323 specifications.', ' It can also be used for fax communication over IP networks (as defined in T.38 specification).', ' G.711, also known as Pulse Code Modulation (PCM), is a very commonly used waveform codec.', ' G.711 is a narrowband audio codec that provides toll-quality audio at 64 kbit/s.', ' G.711 passes audio signals in the range of 300–3400\\xa0Hz and samples them at the rate of 8,000 samples per second, with the tolerance on that rate of 50 parts per million (ppm).', ' Non-uniform (logarithmic) quantization with 8 bits is used to represent each sample, resulting in a 64 kbit/s bit rate.', ' There are two slightly different versions: μ-law, which is used primarily in North America, and A-law, which is in use in most other countries outside North America.']], ['Background extinction rate', [\"Background extinction rate, also known as the normal extinction rate, refers to the standard rate of extinction in earth's geological and biological history before humans became a primary contributor to extinctions.\", ' This is primarily the pre-human extinction rates during periods in between major extinction events.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.707\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a86edcc55429960ec39b6da', 'answer': 'Jennifer Aniston', 'question': 'What actress portrayed Rachel Green on the sitcom \"Friends\" and starred in the movie \"She\\'s Funny That Way\"?', 'supporting_facts': [[\"She's Funny That Way (film)\", 1], ['Jennifer Aniston', 2]], 'context': [['Ross Geller', ['Dr. Ross Eustace Geller, Ph.D., is a fictional character from the NBC sitcom \"Friends\", portrayed by David Schwimmer.', ' Ross is considered by many to be the most intelligent member of the group and is noted for his goofy, pathetic but lovable demeanor.', ' He is a compulsive liar to avoid arguments or situations with conflict, often leading to an arcing storyline within a show.', ' His relationship with Rachel Green was included in \"TV Guide\"\\' s list of the best TV couples of all time, as well as \"Entertainment Weekly\"\\' s \"30 Best \\'Will They/Won\\'t They?\\'', ' TV Couples\".', ' \\xa0Kevin Bright, one of the executive producers of the show had worked\\xa0with Schwimmer before, so the writers were already developing Ross’s character in Schwimmer’s voice.', ' And hence, Schwimmer was the first person to be cast on the show.']], ['Jennifer Aniston', ['Jennifer Joanna Aniston (born February 11, 1969) is an American actress, producer, and businesswoman.', ' She is the daughter of Greek-born actor John Aniston and American actress Nancy Dow.', ' Aniston gained worldwide recognition for portraying Rachel Green on the television sitcom \"Friends\" (1994–2004), a role which earned her a Primetime Emmy Award, a Golden Globe Award, and a Screen Actors Guild Award.', ' The character was widely popular during the airing of the series and was later recognized as one of the 100 greatest female characters in United States television.']], ['Meghan Markle', ['Rachel Meghan Markle (born August 4, 1981), is an American actress, model and humanitarian from Los Angeles.', ' Since 2011 she has portrayed Rachel Zane on the legal drama series \"Suits\" and is also known for her work as FBI special agent Amy Jessup in the sci-fi thriller \"Fringe\".']], ['Chandler Bing', [\"Chandler's best friend is Ross Geller from the college.\", ' He and Ross were in a band named Way/No Way during college.', \" He met Ross's sister, Monica Geller, and her friend, Rachel Green, while celebrating Thanksgiving at Ross's House.\", \" Chandler was the first person to know about Ross's love for Rachel.\", ' He moved to New York City and lives across the hall from Monica and, through her, meets Phoebe Buffay.', ' At some point during this time, Joey Tribbiani moved in with him and they became the best friends throughout the journey.', ' Chandler has a very good sense of humor, and is notoriously sarcastic.', ' He is personally the highest earning member of his friends due to responsible income management, and learning the value of money from a young age.', ' He suffers from the commitment issues but later on, he ended up marrying Monica.', ' He is afraid of dogs.']], [\"She's Funny That Way (film)\", ['She’s Funny That Way is a 2014 American screwball comedy film directed by Peter Bogdanovich and co-written with Louise Stratten.', ' The film stars Owen Wilson, Imogen Poots, Kathryn Hahn, Will Forte, Rhys Ifans and Jennifer Aniston.']], ['List of Friends characters', ['Various characters appeared in the sitcom \"Friends\", which aired for ten seasons on NBC from 1994 to 2004.', ' It featured six main cast members: Rachel Green (Jennifer Aniston), Phoebe Buffay (Lisa Kudrow), Joey Tribbiani (Matt LeBlanc), Chandler Bing (Matthew Perry), Ross Geller (David Schwimmer) and Monica Geller (Courteney Cox).', ' Many celebrities guest starred on the series throughout its ten-year run.']], ['The One with the Rumor', ['\"The One with the Rumor\" is the ninth episode of the eighth season of the American television situation comedy \"Friends\", which aired on NBC on November 22, 2001.', ' It continues the series\\' annual Thanksgiving-themed episode tradition, and guest-stars cast member Jennifer Aniston\\'s then-husband Brad Pitt in the uncredited role of Will Colbert, who reveals that he and Ross (David Schwimmer) were part of an \"I hate Rachel\" club.', ' The two of them hated Rachel Green (Aniston) and got the exchange student from Thailand to join their club.', ' Will revealed that they spread a rumor in high school that Rachel (Aniston) was a hermaphrodite.']], ['Joey Tribbiani', ['Joseph Francis \"Joey\" Tribbiani, Jr. is a fictional character from the NBC sitcoms \"Friends\" and its spin-off \"Joey\", portrayed by Matt LeBlanc.', ' An Italian-American struggling actor, he lives in New York City with his roommate and best friend, Chandler Bing, and hangs out in a tight-knit group of friends - Chandler Bing, Ross Geller, Monica Geller-Bing, Rachel Green and Phoebe Buffay.']], ['Zen Gesner', ['Zen Brant Gesner (born June 23, 1970) is an American television and movie actor.', ' He is perhaps most recognized for his roles as Sinbad in the syndicated television series \"The Adventures of Sinbad\", and was a regular cast member on the ABC daytime drama \"All My Children\" as bad boy and rapist Braden Lavery.', ' More recently he\\'s appeared in Miller Lite\\'s \"Man Laws\" commercials as one of the \"Men Of The Square Table\".', ' Gesner also appeared on an episode of the popular sitcom \"Friends\" in which he played Rachel Green\\'s date.', ' A graduate of the prestigious London Academy of Music and Dramatic Art (LAMDA), Gesner has appeared in several movies since his cinematic debut as \"Dale\\'s Man #1\" in the 1994 comedy \"Dumb & Dumber\", including \"Osmosis Jones\" (as Emergency Room Doctor #1), \"Me, Myself & Irene\" (Agent Peterson), \"Shallow Hal\" (Ralph), and \"There\\'s Something About Mary\" (as a bartender).', ' In 2005, he had a small part in the romantic comedy \"Perfect Catch\" starring Drew Barrymore and Jimmy Fallon.']], ['List of Jennifer Aniston performances', ['American actress Jennifer Aniston made her screen debut in the television series \" Molloy\" (1990).', ' Her film career began in the horror film \"Leprechaun\" (1993).', ' She gained worldwide recognition in the 1990s for portraying Rachel Green on the television sitcom \"Friends\" (1994–2004), a role which earned her an Emmy Award, a Golden Globe Award, and a Screen Actors Guild Award.', ' In 2012, she received a star on the Hollywood Walk of Fame.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.707\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae122465542997b2ef7d0f4', 'answer': 'the Allies and Nazi Germany', 'question': 'The uncle of Paul Capellani died at a military operation during a battle was fought between who?', 'supporting_facts': [['Paul Capellani', 0], ['Paul Capellani', 1], ['Battle of Dunkirk', 0], ['Battle of Dunkirk', 1]], 'context': [['Operation Mersad', ['Operation Mersad (Persian: عملیات مرصاد\\u200e \\u200e , meaning \"ambush\") was the last major military operation of the Iran–Iraq War, ending in a decisive victory for Iran.', ' The operation involved a successful counterattack against a July 1988 military incursion from Iraq, by a military force of about 7,000 members of the Mujahadeen-e-Khalq (MEK).', ' The MEK soldiers were armed, equipped and given air support by Iraq.', ' Led by Lt. General Ali Sayad Shirazi, Operation Mersad began on 26 July 1988 and lasted only a few days, where the Iranian Armed Forces crushed the MEK in what was the last military operation of any significance of the war.']], ['Paul Capellani', ['Paul Capellani (September 9, 1877 – November 7, 1960) was a noted French silent film actor.', ' His brother was the director Albert Capellani and his uncle the film director Roger Capellani who died May 1940 at the Battle of Dunkirk.']], ['Albert Capellani', ['Albert Capellani (23 August 1874 – 26 September 1931) was a French film director and screenwriter of the silent era.', ' He directed films between 1905 and 1922.', ' One of his brother was the actor-sculptor Paul Capellani.', ' and another the film director Roger Capellani.']], ['Roger la Honte (1913 film)', [\"Roger la Honte or A Man's Shadow is a 1913 French silent historical drama film directed by Adrien Caillard and starring Georges Dorival, Paul Capellani and Henri Collen.\", ' It is an adaption of the novel of the same title by Jules Mary, which has been filmed a further four times since.']], ['Battle of Dunkirk', ['The Battle of Dunkirk was a military operation that took place in Dunkirk (Dunkerque), France, during the Second World War.', ' The battle was fought between the Allies and Nazi Germany.', ' As part of the Battle of France on the Western Front, the Battle of Dunkirk was the defence and evacuation of British and Allied forces in Europe from 26 May to 4 June 1940.']], ['Operation Balavegaya', ['Operation Balavegaya (Operation Power force) was a combined military operation launched by the Sri Lankan military in Jaffna, the largest amphibious assault in its history.', ' Operation Balavegaya was launched in response to the siege of Elephant Pass by the LTTE.', ' It is believed that Operation Balavegaya was the largest and most successful military operation of the Sri Lankan military until Operation Riviresa in 1995.']], ['Roger Capellani', ['Roger Capellani (31 January 1905 – 30 May 1940) was a French film director, the son of film director and screenwriter Albert Capellani and the nephew of the actor Paul Capellani.']], ['La Bohème (1916 film)', ['La Bohème (aka:La vie de Bohème) is a 1916 silent historical film directed by Albert Capellani and distributed by World Pictures.', ' The star of this version is Alice Brady, whose father William A. Brady was the founder of World Pictures.', ' This film is one of many silent versions, actually the third or fourth.', ' Later silent versions appeared in 1917 and 1926 starring Lillian Gish.', \" Director Albert Capellani's brother, Paul Capellani, who appears in this film, had made his own short version in 1912.\"]], ['Camille (1915 film)', ['Camille is a 1915 American silent film based on the story \"La Dame aux Camélias\" (\"The Lady of the Camellias\") by Alexandre Dumas, \"fils\", first published in French as a novel in 1848 and as a play in 1852.', ' Adapted for the screen by Frances Marion, \"Camille\" was directed by Albert Capellani and starred Clara Kimball Young as Camille and Paul Capellani as her lover, Armand.']], ['Patrie (1917 film)', ['Patrie is a 1917 French film by Albert Capellani after the drama of Victorien Sardou.', ' The film featured Henry Krauss as the Count of Rysoor, Paul Capellani as Karloo Van der Noot, Léon Bernard as Ionas, and Maxime Desjardins as the Duke of Alba.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.708\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae3fd595542995dadf24292', 'answer': 'Thriller', 'question': 'Arrêtez-moi is a film in which film genre, known for suspense and anxiety?', 'supporting_facts': [['Arrêtez-moi', 0], ['Thriller (genre)', 0], ['Thriller (genre)', 1]], 'context': [['Thriller film', ['Thriller film, also known as suspense film or suspense thriller, is a broad film genre that invokes excitement and suspense in the audience.', \" The suspense element, found in most films' plots, is particularly exploited by the filmmaker in this genre.\", ' Tension is created by delaying what the audience sees as inevitable, and is built through situations that are menacing or where escape seems impossible.']], [\"Woman's film\", [\"The woman's film is a film genre which includes women-centered narratives, female protagonists and is designed to appeal to a female audience.\", ' Woman\\'s films usually portray \"women\\'s concerns\" such as problems revolving around domestic life, the family, motherhood, self-sacrifice, and romance.', ' These films were produced from the silent era through the 1950s and early 1960s, but were most popular in the 1930s and 1940s, reaching their zenith during World War II.', \" Although Hollywood continued to make films characterized by some of the elements of the traditional woman's film in the second half of the 20th century, the term itself disappeared in the 1960s.\", \" The work of directors George Cukor, Douglas Sirk, Max Ophüls, and Josef von Sternberg has been associated with the woman's film genre.\", \" Joan Crawford, Bette Davis, and Barbara Stanwyck were some of the genre's most prolific stars.\"]], ['Film genre', ['A film genre is a motion picture category based on similarities in either the narrative elements or the emotional response to the film (namely, serious, comic, etc.).', ' Most theories of film genre are borrowed from literary genre criticism.', ' The basic genres include fiction and documentary, from which subgenres have emerged, such as docufiction and docudrama.', ' Other subgenres include the courtroom and trial-focused drama known as the legal drama.', ' Types of fiction which may seem unrelated can also be combined to form hybrid subgenres, such as the melding of horror and comedy in the \"Evil Dead\" films.', ' Other popular combinations are the romantic comedy and the action comedy film.']], ['Singapore Short Film Awards', ['The Singapore Short Film Awards (abbreviation: SSFA) is an annual event which promotes and recognises excellence in short films in Singapore.', \" It began in 2010 and was jointly organised by The Substation and Objectifs, presented by The Substation's Moving Images.\", \" Created by filmmaker Chai Yee Wei, former Programme Manager of The Substation's Moving Images Low Beng Kheng and current Co-Founder of Objectifs Yuni Hadi, the Singapore Short Film Awards highlights quality work done annually in the short film genre in Singapore - by seeking out new talent, reflecting current standards of the short film genre and to bring together both the veterans and the young talents as a community to create a space for networking and sharing.\"]], ['Actuality film', ['The actuality film is a non-fiction film genre that, like the documentary film, uses footage of real events, places, and things, yet unlike the documentary is not structured into a larger argument, picture of the phenomenon or coherent whole.', ' In practice, actuality films preceded the emergence of the documentary.', ' During the era of early cinema, actualities—usually lasting no more than a minute or two and usually assembled together into a program by an exhibitor—were just as popular and prominent as their fictional counterparts.', ' The line between \"fact\" and \"fiction\" was not so sharply drawn in early cinema as it would become after the documentary came to serve as the predominant non-fiction filmmaking form.', ' An actuality film is not like a newspaper article so much as it is like the still photograph that is published along with the article, with the major difference being that it moves.', ' Apart from the traveling actuality genre, actuality is one film genre that remains strongly related to still photography.']], ['Arrêtez-moi', ['Arrêtez-moi (English: \"Stop Me\" ) is a 2013 French thriller film directed by Jean-Paul Lilienfeld and starring Sophie Marceau, Miou-Miou, and Marc Barbé.', ' Written by Jean-Paul Lilienfeld and Jean Teulé, the film is about a woman who shows up at a police station and confesses to the murder of her abusive husband several years earlier.', ' The female police officer who interviews her cannot understand why this woman who was never a suspect has come forward after all this time.', \" The more she learns about the woman's life, the less she wants to arrest her.\", ' \"Arrêtez-moi\" was released on 6 February 2013 in France.']], ['OP Eiga', [\"OP Eiga (オーピー映画 ) , also known as Ōkura Eiga (大蔵映画 ) is the largest and one of the oldest independent Japanese studios which produce and distribute pink films. Along with Shintōhō Eiga, Kantō, Million Film, and Kōji Wakamatsu's production studio, Ōkura was one of the most influential studios on the pink film genre.\", ' Among the many notable pink films released by the studio are Satoru Kobayashi\\'s \"Flesh Market\" (1962), the first film in the pink film genre.']], ['Mexican sex comedy', ['The Mexican sex comedies film genre, generally known as Ficheras film or Sexicomedias is a genre of sexploitation and Mexploitation films of the Mexican Cinema that flourished in the 1970s and 1980s.', ' It is recognized as a collection of usually low quality films with low budgets.', ' Although the films had sexual tones and used double entendre, they were not particularly explicit.', ' The genre is possibly based on the Italian erotic comedies.', ' The popular term for it came from the film \"Las ficheras,\" produced and released in 1975, which described the experiences of many women who entertained men at nightclubs.']], ['Internet genre', ['Internet genre refers to a type of genre ( or ) explored in multimedia Studies.', ' Others include film genre, video game genres and music genre.', ' Genre, in terms of genre studies refers to the method based on similarities in the narrative elements from which media-texts are constructed.']], ['Thriller (genre)', ['Thriller is a broad genre of literature, film and television, having numerous, often overlapping subgenres.', ' Thrillers are characterized and defined by the moods they elicit, giving viewers heightened feelings of suspense, excitement, surprise, anticipation and anxiety.', ' Successful examples of thrillers are the films of Alfred Hitchcock.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.709\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a829c1d55429966c78a6a67', 'answer': 'court in Åbo', 'question': \"What did  the man who  led Sweden to military supremacy during the Thirty Years' War found in 1623?\", 'supporting_facts': [['Hovrätt', 2], ['Gustavus Adolphus of Sweden', 1]], 'context': [['Johann von Geyso', [\"Johann von Geyso (1593 – 1661) was a German nobleman and General-Lieutenant, who fought during the course of the Thirty Years' War.\", ' After studying in a Dutch military academy, Geyso fought as a mercenary in the armies of Sweden, Bohemia, Denmark and the German Protestant Union.', \" In 1628, having gained significant experience in warfare he returned to his native Hesse-Kassel which he served until the end of the Thirty Years' War, reaching the rank of commander in chief of the Langraviate's forces and becoming ennobled.\"]], ['Wallenstein (novel)', ['Wallenstein is a 1920 historical novel by German author Alfred Döblin.', \" Set in Central Europe during the Thirty Years War, the novel's plot is organized around the polar figures of Ferdinand II, Holy Roman Emperor, on the one hand, and Albrecht von Wallenstein, on the other.\", \" Döblin's approach to narrating the war differed from prevailing historiography in that, rather than interpreting the Thirty Years War primarily as a religious conflict, he portrays it critically as the absurd consequence of a combination of national-political, financial, and individual psychological factors.\", ' Döblin saw a strong similarity between the Thirty Years War and the First World War, during which he wrote \"Wallenstein\".', ' The novel is counted among the most innovative and significant historical novels in the German literary tradition.', ' In large part, contemporary critics found the novel to be difficult, dense, and chaotic—a reception Döblin discussed in his 1921 essay \"The Epic Writer, His Material, and Criticism\"—yet writers such as Lion Feuchtwanger, Franz Blei, and Herbert Ihering praised \"Wallenstein\" for its formal innovation, poetic language, epic scope, and bold departure from other German writing of the time.', ' Despite the novel\\'s difficulty, the critical consensus was that \"Wallenstein\" was a major achievement and confirmed the promise seen in Döblin\\'s earlier historical novel, \"The Three Leaps of Wang Lun\".']], ['Hovrätt', ['Hovrätt (Finnish: Hovioikeus ) (literally \"Royal Court\") was the highest judicial body in Sweden until King Gustav III founded the Supreme Court of Sweden in 1789.', ' The first hovrätt, Svea hovrätt, was founded 1614 in Stockholm.', ' In Finland, then a part of Sweden, the court in Åbo was founded in 1623 by Gustavus Adolphus, mainly due to the distance to Stockholm.', ' Today, these courts mostly function as an appellate court, the second highest judicial body in both Sweden and Finland.']], [\"Swedish intervention in the Thirty Years' War\", [\"The Swedish invasion of the Holy Roman Empire, or the Swedish Intervention in the Thirty Years' War is a historically accepted division of the Thirty Years' War.\", \" It was a military conflict that took place between 1630 and 1635, during the course of the Thirty Years' War.\", ' It was a major turning point of the war, as during this time, the Protestant cause, previously on the verge of defeat, won several major victories and snatched victory away from the Habsburg-Catholic coalition.', ' It is often considered to be an independent conflict by most historians.']], ['The Last Valley (novel)', [\"The Last Valley (1959), by J. B. Pick, is an historical novel about the Thirty Years' War (1618–1648).\", ' The story occurs from September 1637 to March 1638, and centres on two men – a mercenary soldier and an intellectual – who are fleeing the destruction and starvation wrought by religious war.', ' In southern Germany, each man stumbles upon a fertile valley untouched by the war.', ' Soldier and intellectual, man of arms and man of mind, must collaborate to preserve the peace and plenty of the last valley from the stress and strain of the religious bigotry that caused thirty years of war in Europe.']], ['John Ruthven (general)', [\"John Ruthven was a military officer who served in Denmark and Sweden during the Thirty Years' War before returning for brief service in the British Civil Wars.\", ' He served first as a captain in Danish service from 1627.', ' As King Christian IV of Denmark-Norway made peace with the Habsburg Emperor in 1629 Ruthven, along with many other Scottish soldiers in Danish service, then turned to Sweden to continue the war.', ' He first appears in Swedish service in 1629 serving as a captain of the Scottish infantry at Stralsund under the command of Alexander Leslie.', \" He was soon promoted lieutenant-colonel in Leslie's infantry regiment (by 1630) and led an infantry-regiment in the battle of Breitenfeld on 17 September 1631 as full colonel.\", ' He later took part in the battle at the Alte Veste near Nuernberg on 3 September 1632, and later took part in the bloody conquest of Landsberg/Lech (Bavaria) under the command of Lennart Torstensson.']], ['Military history of Iran', ['With thousands of years of recorded history, and due to an unchanging geographic (and subsequently geopolitical) condition, Iran (previously known as Persia in the West until 1935) has had a long, varied, and checkered military culture and history, ranging from triumphant and unchallenged ancient military supremacy affording effective superpower status in its day, to a series of near catastrophic defeats (beginning with the destruction of Elam) at the hand of previously subdued and conquered peripheral nations (including Greece, Macedon and the Asiatic nomadic tribes at the Eastern boundary of the lands traditionally home to the Iranian people).']], ['Hakkapeliittain Marssi', ['Hakkapeliittain marssi (\"March of the Hakkapeliittas\") or Finska Rytteriets Marsch \"in Swedish\" (\"March of the Finnish Cavalry\"), also known as Suomalaisen ratsuväen marssi 30-vuotisessa sodassa or Finska rytteriets marsch i trettioåriga kriget (\"March of the Finnish cavalry in 30 years war\") is one of the Finnish and Swedish cavalry\\'s battle marches and one of the oldest currently played.', \" It originates from the times of Thirty Years' War when Finnish cavalrymen were known as hakkapeliitta and it became popular with military bands.\", ' It was given lyrics (in Swedish) in 1872 by Zacharias Topelius and is commonly known as the \"March of the Finnish Cavalry during the Thirty Years War\".', ' The Prussian army officially adopted it for use in 1891; it is now a standard of the German marching band repertoire.']], ['Charles X Gustav of Sweden', ['Charles X Gustav, also Carl Gustav (Swedish: \"Karl X Gustav\" ; 8 November 1622 – 13 February 1660), was King of Sweden from 1654 until his death.', ' He was the son of John Casimir, Count Palatine of Zweibrücken-Kleeburg and Catherine of Sweden.', \" After his father's death he also succeeded him as Pfalzgraf.\", ' He was married to Hedwig Eleonora of Holstein-Gottorp, who bore his son and successor, Charles XI.', ' Charles X Gustav was the second Wittelsbach king of Sweden after the childless king Christopher of Bavaria (1441–1448) and he was the first king of the Swedish \"Caroline era\", which had its peak during the end of the reign of his son, Charles XI.', ' He led Sweden during the Second Northern War, enlarging the Swedish Empire.', ' By his predecessor Christina, he was considered \"de facto\" Duke of Eyland (Öland) before ascending to the Swedish throne.']], ['Gustavus Adolphus of Sweden', ['Gustav II Adolf (9 December 1594 – 6 November 1632, O.S.), widely known in English by his Latinised name Gustavus Adolphus or as Gustav II Adolph, was the King of Sweden from 1611 to 1632 and is credited as the founder of Sweden as a Great Power (Swedish: \"Stormaktstiden\" ).', \" He led Sweden to military supremacy during the Thirty Years' War, helping to determine the political as well as the religious balance of power in Europe.\", ' He was formally and posthumously given the name Gustavus Adolphus the Great (Swedish: \"Gustav Adolf den store\" , Latin: \"Gustavus Adolphus Magnus\" ) by the Riksdag of the Estates in 1634.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.709\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a89e47f5542992e4fca8457', 'answer': '9,000', 'question': 'The university where Rolf Martin Theodor Dahlgren worked as a professor has how many employees?', 'supporting_facts': [['Rolf Dahlgren', 0], ['University of Copenhagen', 2]], 'context': [['Gertrud Dahlgren', ['Gertrud Dahlgren (1931–2009) was a Swedish botanist.', ' She was born at Klippan in Scania, and after graduating from university at Helsingborg went to the University of Lund for graduate studies.', ' There she obtained her M.Sc.', ' in chemistry and biology.', ' She was married to fellow botanist Rolf Dahlgren (1932–1987), who was killed in a car accident.', ' The couple had three children, Susanna, Helena and Fredrik.', ' She died in December 2009 at the age of 78.']], ['Al Khor (city)', ['Al Khor is a coastal city in northern Qatar, located 50 kilometres north of the capital, Doha.', ' It is the capital city of the municipality of Al Khor.', ' The name of the city means creek in Arabic as the town is located on a creek.', \" Al Khor is home to many employees of the oil industry due to its proximity to Qatar's northern oil and natural gas fields, and due to its proximity to the Ras Laffan Industrial City.\"]], ['Theodor von Heuglin', ['Martin Theodor von Heuglin (20 March 1824, Hirschlanden, Württemberg – 5 November 1876), was a German explorer and ornithologist.']], ['Anker Hagen', ['Rolf Martin Anker Hagen (born 22 October 1920) is a Norwegian former sport shooter who competed in the 1956 Summer Olympics in Melbourne.', ' Participating in two events, he finished 13th in a field of 44 shooters in the 50 metre rifle prone competition and 30th among 44 shooters in the 50 metre rifle three positions competition.', ' A native of Oslo, he also attended the 1954 ISSF World Shooting Championships, where he brought home four medals from the team tournament: silver in the 50 metre free rifle prone and standing 40 shots events and bronze in the 50 metre free rifle kneeling 40 shots and 50 metre rifle three positions events.']], ['New Urban Entertainment', ['New Urban Entertainment Television (NUE-TV) was an American cable network targeted toward African-American audiences.', ' It was a direct competitor to Black Entertainment Television (BET), but was aiming for a more mature audience with more news.', ' It operated between July 17, 2000 and October 31, 2002 and reached close to 3 million subscribers.', ' In 2003, it was permanently shut down due to financial difficulties.', ' A big investor was Radio One and many employees came from BET.']], ['Rolf Dahlgren', ['Rolf Martin Theodor Dahlgren (7 July 1932 – 14 February 1987) was a Swedish-Danish botanist, professor at the University of Copenhagen from 1973 to his death.']], ['University of Copenhagen', ['The University of Copenhagen (UCPH) (Danish: \"Københavns Universitet\" ) is the oldest university and research institution in Denmark.', ' Founded in 1479 as a studium generale, it is the second oldest institution for higher education in Scandinavia after Uppsala University (1477).', ' The university has 23,473 undergraduate students, 17,398 postgraduate students, 2,968 doctoral students and over 9,000 employees.', ' The university has four campuses located in and around Copenhagen, with the headquarters located in central Copenhagen.', ' Most courses are taught in Danish; however, many courses are also offered in English and a few in German.', ' The university has several thousands of foreign students, about half of whom come from Nordic countries.']], ['American Hot Rod', ['American Hot Rod was a reality television series that originally aired between 2004 and 2008 on The Learning Channel and Discovery Channel.', \" The unique series documented the crew at Boyd Coddington's car shop and their personal struggles to build hot rods and custom vehicles.\", \" It was made on location at Coddington's hot rod and wheel shop in La Habra, California.\", ' Many employees went to work for \"Overhaulin\\'\"s Chip Foose, a former partner of Coddington.']], ['Rolf M. Zinkernagel', ['Rolf Martin Zinkernagel {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} (born January 6, 1944 in Riehen, Basel-Stadt, Switzerland) is Professor of Experimental Immunology at the University of Zurich.', ' He was awarded the Nobel Prize in Physiology or Medicine in 1996 for the discovery of how the immune system recognizes virus-infected cells.']], ['Abbalagere', ['Abbalagere was previously a Tobacco growing village, but now farmers are growing more Aracanut and plantation crops.', ' Paddy, maize, vegetables are major crops.', ' Most of farmers in the village are medium land holders.', ' The village is close to Shimoga city, so many employees commute daily between the city and the village for work.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.710\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab41f6b5542991751b4d671', 'answer': '\"I Believe I Can Fly\"', 'question': 'What album did R.Kelly release the same year as producing \"These Are Special Times\" for Celine Dion. ', 'supporting_facts': [['These Are Special Times', 1], ['R. Kelly', 4]], 'context': [[\"I'm Your Angel\", ['\"I\\'m Your Angel\" is a duet by Celine Dion and R. Kelly from Dion\\'s \"These Are Special Times\" album and Kelly\\'s \"R.\" album.', ' It was released on 16 November 1998.', ' The song was written and produced by R. Kelly.', ' The single was very successful, reaching number 1 in the United States and was certified platinum by the RIAA.', ' The single also reached the top 5 in the United Kingdom and Ireland.']], [\"Vois comme c'est beau\", ['\"Vois comme c\\'est beau\" (meaning \"Look How Beautiful it Is\") is a duet between Claudette Dion and her sister, Celine Dion, released as a single from Claudette Dion\\'s album \"Hymnes à l\\'amour: Volume 2\".', ' It was issued in 1985 in Quebec, Canada.', ' \"Vois comme c\\'est beau\" has never appeared on any of Celine Dion\\'s albums.']], ['Tamara Gee', ['Tamara Gee (born Tamara Diane Wimer on October 11, 1972 in Seattle), is an American vocalist, songwriter, producer, arranger, dancer, and model.', ' She grew up singing and performing from the age of 5, and was a professional vocalist by the time she was 12 years old, winning various singing competitions throughout her adolescence and adulthood, as well as a beauty pageant.', ' Gee opened for vocalist Tony Bennett and his orchestra after being the featured vocalist on various albums throughout her teen years.', ' She released her debut album \"Hidden Treasure\" with Universal Music in 2007, while living in Poland.', ' Her single from the album, \"For Life\", was voted unanimous winner of Piosenka dla Europy, the Polish final for the 2008 Eurovision Song Contest.', ' Tamara and her song \"For Life\" was the first Polish entry in history to qualify for the Eurovision final since the new rules were designed in the contest and a French television commentator compared Gee\\'s performance to Celine Dion.', ' A portion of Gee\\'s Eurovision performance was shown on \"The Oprah Winfrey Show\" on \"The World\\'s Got Talent\" episode with Simon Cowell where Gee sang with former participants Celine Dion, ABBA and Julio Iglesias.', ' In 2007 Gee co-wrote and performed \"Fate\" on DJ Schiller\\'s album \"Sehnsucht\".', ' The album went triple platinum and was nominated for a Grammy.', ' She released her solo EP \"Christmas Angel\" in 2009.', ' On November 27, 2014, Tamara released a new highly anticipated album \"Love, Tamara\" of which she wrote with and was produced by Multi Grammy Award winning producer/songwriter Walter Afanasieff (Celine Dion, Whitney Houston, Mariah Carey, etc.).', ' She has been named the next Celine Dion.']], ['Celine Dion albums discography', ['Canadian singer Celine Dion has released twenty-six studio albums, seven live albums, seventeen compilation albums, and twenty-one box sets.', ' Her debut album, \"La voix du bon Dieu\" was issued in 1981.', \" In the '80s, Dion released her French-language albums in Canada, with several compilation albums issued also in France.\", ' Her first English-language album, entitled \"Unison\" was released in 1990 and has sold over three million copies worldwide.', ' It was followed by \"Dion chante Plamondon\" in 1991 and \"Celine Dion\" in 1992.', ' The latter became one of six of her albums to be certified Diamond in Canada for shipments of at least one million units.', ' Dion\\'s popularity became well-established with her 1993 album, \"The Colour of My Love\", which topped the charts in various countries, including the United Kingdom, Canada, and Australia, and has sold twenty million copies around the world.', ' In the United States, it was certified six-times platinum.', ' Released in 1995, \"D\\'eux\" became the best-selling French-language album in history, with sales of ten million copies worldwide.', ' In France alone, \"D\\'eux\" spent forty-four weeks at the top of the chart and has sold 4.5 million units, becoming the best-selling album of all time.', \" It also became Dion's first out of six Diamond-certified albums in France.\"]], ['Celine Dion in Concert', ['Celine Dion in Concert was the fourth concert tour by Celine Dion.', ' The tour consisted of 51 shows held between 13 July 1992 and 13 May 1993.', ' It was organized to support the album \"Celine Dion\".']], ['These Are Special Times', ['These Are Special Times is the sixth English-language studio album and the first English-language Christmas album by Canadian singer Celine Dion.', ' Released by Sony Music Entertainment on 30 October 1998, it features cover versions of popular Christmas tunes and original material.', ' Dion worked with David Foster and Ric Wake, who produced most of the tracks for the album.', ' Other producers include R. Kelly and Bryan Adams.', \" Critics praised Dion's commitment to the recorded material, as well as the production of the songs.\"]], [\"Don't Save It All for Christmas Day\", ['\"Don\\'t Save It All for Christmas Day\" is a song by Canadian recording artist Celine Dion.', ' It was written by Peter Zizzo, Ric Wake, and Dion for her first English-language holiday album \"These Are Special Times\" (1998), while Wake also served as its producer.', \" The pop ballad was issued as a promotional single on 4 December 2000, two years after album's original release.\"]], ['R. Kelly', ['Robert Sylvester Kelly (born January 8, 1967), known professionally as R. Kelly, is an American singer, songwriter, record producer, and former professional basketball player.', ' A native of Chicago, Illinois, Kelly began performing during the late 1980s and debuted in 1992 with the group Public Announcement.', ' In 1993, Kelly went solo with the album \"12 Play\".', ' He is known for a collection of major hit singles including \"Bump N\\' Grind\", \"Your Body\\'s Callin\\'\", \"I Believe I Can Fly\", \"Gotham City\", \"Ignition (Remix)\", \"If I Could Turn Back the Hands of Time\", \"The World\\'s Greatest\", \"I\\'m a Flirt (Remix)\", and the hip-hopera \"Trapped in the Closet\".', ' In 1998, Kelly won three Grammy Awards for \"I Believe I Can Fly\".', ' His distinctive sound and style has influenced numerous hip hop and contemporary R&B artists.', ' Kelly became the first musician to play professional basketball, when he was signed in 1997.']], ['A New Day Has Come (TV special)', ['A New Day Has Come is the third one-off American television special by Canadian singer Celine Dion that was broadcast by CBS on 7 April 2002.', ' The special was a promotion for Dion\\'s first English album in 2 years of the same name, \"A New Day Has Come\".', \" It also marks as Dion's comeback after her 2-year hiatus from the music industry.\", ' The special was filmed on 2 March 2002 at the Kodak Theatre in Los Angeles, California.', ' It featured Dion (backed by her touring band) performing songs from the album as well as some of her greatest hits.', \" She was also joined by special guests Grammy winning R&B singing sensations Destiny's Child and Brian McKnight.\"]], ['These Are Special Times (TV special)', ['These Are Special Times is a one-off American television special by Canadian singer Celine Dion that was broadcast by CBS on 25 November 1998.', ' The special was a promotion for her first English Holiday album of the same name, \"These Are Special Times\".', ' The special was filmed in front of a live studio audience.', ' It featured Dion (backed by her touring band and a full orchestra) performing holiday music from the album as well as some of her hits.', \" She was also joined by special guests comedic actress and singer Rosie O'Donnell and Italian Tenor Andrea Bocelli.\", ' The special also included footage of Dion in her hometown of Charlegmagne, Quebec.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.711\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae0c7b055429945ae95944b', 'answer': 'Elizabethan theatre', 'question': \"The Admiral's Men occupied which kind of theatre in the 1590s ?\", 'supporting_facts': [['Playing company', 4], ['The Rose (theatre)', 0]], 'context': [['2011 Hotan attack', ['The 2011 Hotan attack was a bomb-and-knife attack that occurred in Hotan, Xinjiang, China on July 18, 2011.', \" According to witnesses, the assailants were a group of 18 young Uyghur men who opposed the local government's campaign against the full-face Islamic veil, which had grown popular among older Hotan women in 2009 but were also used in a series of violent crimes.\", ' The men occupied a police station on Nuerbage Street at noon, killing two security guards with knives and bombs and taking eight hostages.', ' The attackers then yelled religious slogans, including ones associated with Jihadism, as they replaced the Chinese flag on top of a police station with another flag, the identity of which is disputed.']], ['The Rose (theatre)', ['The Rose was an Elizabethan theatre.', ' It was the fourth of the public theatres to be built, after The Theatre (1576), the Curtain (1577), and the theatre at Newington Butts (c. 1580?)', \" – and the first of several playhouses to be situated in Bankside, Southwark, in a liberty outside the jurisdiction of the City of London's civic authorities.\"]], ['Playing company', ['In Renaissance London, playing company was the usual term for a company of actors.', ' These companies were organized around a group of ten or so shareholders (or \"sharers\"), who performed in the plays but were also responsible for management.', ' The sharers employed \"hired men\" that is, the minor actors and the workers behind the scenes.', \" The major companies were based at specific theatres in London; the most successful of them, William Shakespeare's company the King's Men, had the open-air Globe Theatre for summer seasons and the enclosed Blackfriars Theatre in the winters.\", \" The Admiral's Men occupied the Rose Theatre in the 1590s, and the Fortune Theatre in the early 17th century.\"]], ['Spanish Golden Age theatre', ['Spanish Golden Age theatre refers to theatre in Spain roughly between 1590 and 1681.', ' Spain emerged as a European power after it was unified by the marriage of Ferdinand II of Aragon and Isabella I of Castile in 1469 and then claimed for Christianity at the Siege of Granada in 1492.', ' The sixteenth and seventeenth centuries saw a monumental increase in the production of live theatre as well as the in importance of the arts within Spanish society.']], [\"Admiral's Men\", [\"The Admiral's Men (also called the Admiral's company, more strictly, the Earl of Nottingham's Men; after 1603, Prince Henry's Men; after 1612, the Elector Palatine's Men or the Palsgrave's Men) was a playing company or troupe of actors in the Elizabethan and Stuart eras.\", \" It is generally considered the second most important acting troupe of English Renaissance theatre (after the company of Shakespeare, the Lord Chamberlain's or King's Men).\"]], [\"Diederichs's stone\", [\"The Diederichs's stone (German: Diederichsstein, ) was a German monument in the city of Qingdao to commemorate Admiral Otto von Diederichs and the German occupation of the Kiautschou Bay concession on November 14, 1897.\", ' The monument was dedicated on November 21, 1898 by Prince Henry of Prussia.', ' It was located at an elevation of 98m, about halfway up the southwestern slope of the Signal Hill, the official German name of the mountain at the time was \"Diederichsberg\" ).', ' Its most prominent feature was a plate decorated with the imperial eagle of the German Empire and the inscription \"For him who won for Kaiser and Reich the land all around, let this rock be named Diederichs\\'s stone\" (\"Der hier für Kaiser warb u. [und] Reich ringsher das Land, nach ihm sei dieser Felsen Diederichsstein genannt\").', ' Below the plate was a rock inscription that read \"In this place on November 14th, 1897, Admiral v.[von] Diederichs took possession of the Kiautschou territory\" (\"Am 14.', ' November 1897 ergriff an dieser Stelle der Admiral v.[von] Diederichs Besitz vom Kiautschou Gebiet\").', ' A separate Chinese inscription was located to the right-hand side of the German text.', ' The entire monument stood about 5\\xa0meters tall.', ' After Japan occupied Qingdao in November 1914, a Japanese inscription (executed using Chinese characters: 大正三年十一月七日, \"November 7 of the third year of the Taishō period\") was placed across the imperial eagle.', ' When Japan handed Qingdao back to the China on December 10, 1922, the monument was dismantled and parts taken to a military museum in Tokyo.']], ['Admiral Theatre', ['The Admiral Theatre in Chicago, Illinois opened in 1927 as a vaudeville house.', ' it was designed by Gallup and Joy and acquired by the Balaban and Katz circuit.', ' The Admiral closed sometime in the late 1950s, and remained shuttered for many years until opening in 1969 as an all-cartoon venue.', ' Unable to draw the crowds necessary to remain open, the Admiral closed again.', ' In the early 1970s, the Admiral was opened as an adult movie house.', ' After receiving a facelift in the 1980s, the Admiral continues to thrive as an adult venue and gentlemen’s club.', ' While the interior has been drastically altered, the facade is in remarkably good shape.']], ['Wongaksa Pagoda', ['Wongaksa Pagoda is a twelve metre high ten storey marble pagoda in the center of Seoul, South Korea.', ' It was constructed in 1467 to form part of Wongaksa temple, that King Sejo had founded two years before on the site of an older Goryeo-period temple, Heungbok-sa.', ' The temple was closed and turned into a kisaeng house by the (later deposed) king known as Yeonsan-gun (1476 – 1506, r. 1494-1506), and under his successor, King Jungjong (1488 – 1544, r.1506–1544) the site was turned into government offices.', ' The pagoda and a memorial stele commemorating the foundation of Wongaksa alone survived.', ' The site of the temple was later occupied by houses.', ' During the Imjin War of the 1590s, the top portion of the pagoda was pulled down and lay on the ground at the foot of the pagoda until it was replaced by American military engineers in 1947.']], ['Shabbethai Horowitz', ['Shabtai Horowitz (Hebrew: שבתי הורוויץ\\u200e \\u200e ; 1590 – 1660) was a rabbi and talmudist, probably born in Ostroh, Volhynia.', ' He was the son of the kabbalist Isaiah Horowitz, and at an early age married the daughter of the wealthy and scholarly Moses Charif of Lublin.', ' With his father he seems to have gone to Prague, where he occupied a position as preacher; from Prague he went as rabbi to Fürth, whence he was called to Frankfurt am Main about 1632, and finally to Vienna about 1650.', ' There he died on April 12, 1660.']], ['Elizabeth FitzGerald, Countess of Lincoln', ['Lady Elizabeth FitzGerald, Countess of Lincoln (1527 – March 1590), also known as The Fair Geraldine, was an Irish noblewoman and a member of the celebrated FitzGerald dynasty.', ' She became the second wife of Sir Anthony Browne and later the third wife of English admiral Edward Clinton, 1st Earl of Lincoln.', ' She was the inspiration for \"The Geraldine\", a sonnet written by Henry Howard, Earl of Surrey.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.711\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8732505542991e77181713', 'answer': 'the Nazi Party', 'question': 'Friedrich Ratzel coined the phrase Lebensraum; which political movement made it notorious by taking it to extremes?', 'supporting_facts': [['Friedrich Ratzel', 0], ['Lebensraum', 2]], 'context': [['Musar movement', ['The Musar movement (also Mussar movement) is a Jewish ethical, educational and cultural movement that developed in the 19th century in Lithuania, particularly among Orthodox Lithuanian Jews.', ' The Hebrew term \"Musar\" (), is from the book of \"Proverbs\" 1:2 meaning moral conduct, instruction or discipline.', ' The term was used by the Musar movement to refer to efforts to further ethical and spiritual discipline.', ' The Musar Movement made significant contributions to Musar literature and Jewish Ethics.']], ['Black Power movement', ['The Black Power movement was a political movement to achieve a form of Black Power and the many philosophies it contains.', ' The movement saw various forms of activism some violent and some peaceful, all hoping to achieve black empowerment.', ' The Black Power movement also represented socialist movements, all with the general motivation of improving the standing of black people in society.', ' Originated during the Civil Rights Movement, some doubted the philosophy of the movement begging for more radical action, taking influences from Malcolm X.', ' The cornerstone of the movement was the Black Panther Party, a Black Power organization dedicated to socialism and the use of violence to achieve it.', ' The Black Power movement developed amidst the criticisms of the Civil Rights Movement in the early 1960s, and over time and into the 1970s, the movement grew and became more violent.', ' After years of violence, many left the movement and the police began arresting violent actors in the movement.', ' The Black Power movement also spilled out into the Caribbean creating the Black Power Revolution.']], ['Rudolf Kjellén', ['Johan Rudolf Kjellén (] , 13 June 1864, Torsö – 14 November 1922, Uppsala) was a Swedish political scientist and politician who first coined the term \"geopolitics\".', ' His work was influenced by Friedrich Ratzel.', ' Along with Alexander von Humboldt, Karl Ritter, and Ratzel, Kjellén would lay the foundations for the German \"Geopolitik\" that would later be espoused prominently by General Karl Haushofer.']], ['Lebensraum', ['The German concept of Lebensraum (] , English: \"living space\" ) refers to policies and practices of settler colonialism which proliferated in Germany from the 1890s to the 1940s.', ' First popularized around 1901, \"Lebensraum\" became a geopolitical goal of Imperial Germany in World War I (1914–1918) originally, as the core element of the \"Septemberprogramm\" of territorial expansion.', '<ref name=\"E/N301\"> </ref> The most extreme form of this ideology was supported by the Nazi Party (NSDAP) and Nazi Germany until the end of World War II.']], ['History of Corsica', ['That the history of Corsica has been influenced by its strategic position at the heart of the western Mediterranean and its maritime routes, only 12 km from Sardinia, 50 km from the Isle of Elba, 80 km from the coast of Tuscany and 200 km from the French port of Nice, was first proposed by the 19th-century German theorist, Friedrich Ratzel.', ' To him is often attributed the description \"mountain in the sea\".', ' Regardless of whether he used that particular phrase the idea is expressed in his magnum opus, \"Anthropogeographie\", which calls Corsica']], ['Al-Ard', ['Al-Ard (Arabic: الارض\\u200e \\u200e , \"The Land\") was a Palestinian political movement made up of Arab citizens of Israel active between 1958 and some time in the 1970s which attracted international attention.', ' Following unsuccessful efforts to secure registration of the organization as an Israeli NGO and secure it a publishing permit, it was outlawed in 1964.', ' The political movement\\'s goal was, according to political historian David McDowall, \"to achieve complete equality and social justice for all classes of people in Israel\" and \"to find a just solution for the Palestine problem as a whole, and as an indivisible unit.\"', \" Al-Ard's disappearance as a movement was linked both to governmental and popular resistance, with the Israeli Community Party denouncing the group and Palestinian Arab communities inside of Israel concerned that Al-Ard might destroy them.\"]], ['Nine-Hour Movement', ['The Nine-Hour Movement started in Canada in 1872, based out of Hamilton, Ontario.', \" This marked Canada's first national attempt at a labour movement, pushing for the nine-hour work day which united both unionized and non-unionized workers alike.\", \" The movement came to its height in May 1872 when a collective force of 1,500 workers demonstrated in Hamilton in a parade-style fashion, which is coined as being the precursor to the traditional holiday of Canada's Labour Day.\", ' Although the movement was an overall failure, as it failed to deliver the nine-hour work day to the majority of work forces and industries, this movement made a major mark in labour relations in Canada.']], ['Gongche Shangshu movement', ['The Gongche Shangshu movement (Traditional Chinese: 公車上書, Simplified Chinese: 公车上书) was a political movement in late Qing dynasty China, seeking reforms and expressing opposition to the Treaty of Shimonoseki in 1895.', ' It is considered the first modern political movement in China.', \" Leaders of the movement later became leaders of the Hundred Days' Reform.\"]], ['Friedrich Ratzel', ['Friedrich Ratzel (August 30, 1844 – August 9, 1904) was a German geographer and ethnographer, notable for first using the term \"Lebensraum\" (\"living space\") in the sense that the National Socialists later would.']], ['Grassroots', ['A grassroots movement (often referenced in the context of a political movement) is one which uses the people in a given district as the basis for a political or economic movement.', ' Grassroots movements and organizations use collective action from the local level to effect change at the local, regional, national, or international level.', ' Grassroots movements are associated with bottom-up, rather than top-down decision making, and are sometimes considered more natural or spontaneous than more traditional power structures.', ' Grassroots movements, using self-organization, encourages community members to contribute by taking responsibility and action for their community.', ' Grassroots movements utilize a variety of strategies from fundraising and registering voters, to simply encouraging political conversation.', ' Goals of specific movements vary, but the movements are consistent in their focus on increasing mass participation in politics.', ' These political movements may begin as small and at the local level, but grassroots politics as Cornel West contends are necessary in shaping progressive politics as they bring public attention to regional political concerns']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.713\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adf24155542992d7e9f92af', 'answer': 'Oberkommando der Wehrmacht', 'question': 'Richard Münch portrayed the German general who served in what capacity during WWII in the 1970 movie Patton?', 'supporting_facts': [['Richard Münch (actor)', 0], ['Alfred Jodl', 0]], 'context': [['Donald Shebib', ['Donald Shebib (born 17 January 1938, Toronto), often called Don Shebib, is a Canadian film director, writer, producer and editor.', ' A graduate of UCLA film school, Shebib gained prominence and critical acclaim in Canadian cinema for his seminal 1970 movie \"Goin\\' Down the Road\", which combined narrative storytelling with Canadian documentary tradition influenced by the British.', ' The low-budget film crew travelled around Toronto in a station wagon, supported by funding from the newly-formed Canadian Film Development Corporation.', ' The movie was screened in New York and hailed by Pauline Kael and Roger Ebert.', ' Kael wrote that the movie showed up the ostensibly forced sincerity and perceived honesty of the films of John Cassavetes.', ' Shebib is the father of Noah \"40\" Shebib.', \" Goin' Down the Road was digitally remastered as one of the key films in the Canadian film canon and was honoured with a screening at the Art Gallery of Ontario.\", ' A sequel was shot in 2011, called \"Goin\\' Down the Road Again,\" featuring some of the original cast members as well as a new generation of characters.']], ['Richard Münch (sociologist)', ['Richard Münch (born 13 May 1945 in Niefern near Pforzheim, Germany) is a German sociologist and, as of 2013, professor emeritus at the University of Bamberg.', ' His primary field is sociological theory, in particular the work of Talcott Parsons.', ' In the 1980s, he was instrumental in popularizing Parsons in Germany and defended his functionalist \"grand theory\" of action against competing approaches, such as rational choice and Niklas Luhmann\\'s systems theory, which had been gaining ground since the 1970s.']], ['Dorian Gray (1970 film)', ['Dorian Gray (Italian: \"Il dio chiamato Dorian\") aka \"The Sins of Dorian Gray\" is a 1970 movie adaptation of Oscar Wilde\\'s novel \"The Picture of Dorian Gray\" starring Helmut Berger.', ' The Italian title translates as \"A God Called Dorian\".']], ['Wilhelm Burgdorf', ['Wilhelm Emanuel Burgdorf (15 February 1895 – 2 May 1945) was a German general in the Wehrmacht during World War II, who served as a commander and staff officer in the German Army (Wehrmacht) (army).', ' In October 1944, Burgdorf assumed the role of the Chief of the Army Personnel Office (\"Heerespersonalamt\") and Chief Adjutant to Adolf Hitler.', ' In this capacity, he played a role in the forced suicide of Field Marshal Erwin Rommel.', ' Burgdorf committed suicide in the \"Führerbunker\" on 2 May 1945 at the conclusion of the Battle of Berlin.']], ['Alfred Jodl', ['Alfred Josef Ferdinand Jodl (   ; 10 May 1890 – 16 October 1946) was a German general and war criminal during World War II, who served as the Chief of the Operations Staff of the Armed Forces High Command (Oberkommando der Wehrmacht).']], ['Richard Münch (actor)', ['Richard Heinrich Ludwig Münch (10 January 1916 – 6 June 1987), better known as Richard Münch, was a German actor, best known for portraying Alfred Jodl in \"Patton\" (1970).', ' He also portrayed General Erich Marcks in \"The Longest Day\" (1962).']], ['The Last Days of Patton', ['The Last Days of Patton is a 1986 made-for-television film sequel to the 1970 film \"Patton\", which portrays the last few months of the general\\'s life.', \" George C. Scott reprises the role of General George S. Patton, and Eva Marie Saint portrays Beatrice Patton, the general's wife.\", ' It was directed by Delbert Mann.']], ['Johannes Steinhoff', ['Johannes \"Macky\" Steinhoff (15 September 1913 – 21 February 1994) was a German general, NATO official and Luftwaffe fighter ace during World War II.', \" He joined the West German government's Rearmament Office as a consultant on military aviation in 1952 and became one of the principal officials tasked with building the German Air Force during the Cold War.\", ' He became the German Military Representative to the NATO Military Committee in 1960, served as Acting Commander Allied Air Forces Central Europe in NATO 1965–1966, as Inspector of the Air Force 1966–1970 and as Chairman of the NATO Military Committee 1971–1974.', ' In retirement, Steinhoff became a widely read author of books on German military aviation during the Second World War and the experiences of the German people at that time.']], ['Ludwig Beck', ['Ludwig August Theodor Beck (29 June 1880\\xa0– 21 July 1944) was a German general and Chief of the German General Staff during the early years of the Nazi regime in Germany before World War II.', \" Ludwig Beck was never a member of the Nazi Party, though in the early 1930s he supported Adolf Hitler's forceful denunciation of the Versailles Treaty and belief in the need for Germany to rearm.\", ' Beck had grave misgivings regarding the Nazi demand that all German officers swear an oath of fealty to the person of Hitler in 1934, though he believed that Germany needed strong government and that Hitler could successfully provide this so long as he was influenced by traditional elements within the military rather than the SA and SS.']], ['From Denver to L.A.', ['\"From Denver to L.A.\" is a song sung by Elton John, appearing on the soundtrack of the 1970 movie, \"The Games\".', ' The song was released as a single in the U.S. in July 1970, miscredited on the record label to \"Elton Johns\".', \" The single was issued just as John's career was starting to take off, but was quickly withdrawn because both John and his then-current record company objected to its release.\", \" It is now an extremely rare collectors' item.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.713\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7fa97d5542994857a767a9', 'answer': 'Rick Ross', 'question': 'Felicia Pearson is a rapper who appeared in the single \"Here I Am\" by who?', 'supporting_facts': [['Here I Am (Rick Ross song)', 0], ['Here I Am (Rick Ross song)', 2], ['Felicia Pearson', 0]], 'context': [['You Can Do It', ['\"You Can Do It\" is a single by American rapper Ice Cube.', ' It was released from the \"Next Friday\" soundtrack.', \" The song features Ice Cube's Westside Connection bandmate Mack 10, as well as female rapper Ms. Toi.\", ' \"You Can Do It\" later appeared on Cube\\'s sixth studio album, \"War & Peace Vol.', ' 2 (The Peace Disc)\" as well as on his \"Greatest Hits\" and \"In the Movies\" compilations.', ' It would also appear on the double platinum \"Save the Last Dance\".']], ['Memphis Bleek discography', ['The discography of American rapper Memphis Bleek consists of four studio albums, twelve mixtapes, seventeen singles (including four as a featured artist) and fifteen music videos.', ' An early signing to rapper Jay-Z\\'s record label Roc-A-Fella Records, Bleek appeared on several of Jay-Z\\'s early songs, and collaborated with him on the single \"It\\'s Alright\" from the soundtrack to the film \"Streets Is Watching\", which peaked at number 61 on the US \"Billboard\" Hot 100.', ' Bleek went on to release his debut album, \"Coming of Age\", in 1999, which contained the singles \"Memphis Bleek Is...\", \"My Hood to Your Hood\" and \"What You Think of That\".']], ['Here We Come (song)', ['\"Here We Come\" is a song American producer/rapper Timbaland.', ' It features frequent collaborators Missy Elliott and Magoo and serves as the lead single for Timbaland\\'s solo debut album, \"\" (1998).', ' The song also features background vocals by Playa and Darryl Pearson.', ' While the song charted and was released via radio airplay on November 17, 1998, it was not granted a physical release in the United States until March 2, 1999; and in October 5, 1999 for Germany.']], ['Felicia Pearson', ['Felicia Pearson (born May 18, 1980) is an American actress, author, and rapper.', ' She is best known for playing a character of the same name, Felicia \"Snoop\" Pearson, on \"The Wire\".', ' She wrote a memoir titled \"Grace After Midnight\" detailing her troubled childhood and time spent in prison for second degree murder.']], ['DJ Klever', ['Josh Winkler (born July 4, 1977), better known as DJ Klever is a prominent American turntablism and 2 time US Disco Mix Club champion.', ' He has toured across the world and resides in Atlanta, Georgia.', \" Since early 2014 he has been rapper Yelawolf's touring DJ, he later started working with the rapper and signed to his record label Slumerican.\", ' He is involved in the rapper\\'s second studio album \"Love Story\", and has appeared to most of the single\\'s music videos.']], ['S.mouse', ['Shwayne Booth Jr., known as S.mouse (later as S.mouse!)', ', is a fictional character from the Australian mockumentary television series, \"Angry Boys\".', ' The character is portrayed by Chris Lilley, who wears a curly wig and blackface makeup.', ' He has appeared in \"Angry Boys\" since the second episode.', ' He is an African American rapper who lives in Calabasas, California.', ' In the second episode, S.mouse is introduced as the rapper who just released the biggest selling hip-hop single of all time, \"Slap My Elbow\".', ' Later in the show, S.mouse is put under house arrest at his parent\\'s home for two months, after defecating on a police car in the music video for his single \"Poo on You\".', ' He also gets dropped from his record label after another one of his videos, Grandmother Fucker, receives many complaints.']], ['Here I Am (Rick Ross song)', ['\"Here I Am\" is the third single from Rick Ross\\'s second album \"Trilla\".', ' It features Nelly and Avery Storm.', ' This song, produced by Drumma Boy, contains an interpolation of Stevie Wonder\\'s \"Lately.\"', ' Video appearances made are DJ Khaled, Birdman, Pitbull, Dre, Felicia Pearson, Ace Hood and others.', ' After the song ends, the video cuts to Ross\\'s other song, \"Maybach Music\".', ' The single debuted at 99 on the Billboard Hot 100, and has peaked at #41.', ' The original third single was going to be \"Luxury Tax\", but it was changed to \"Here I Am\".']], ['We Want Eazy', ['\"We Want Eazy\" is a single by the American gangsta rapper Eazy-E, from his 1988 debut album, \"Eazy-Duz-It\".', ' The song features fellow N.W.A members Dr. Dre and MC Ren and was produced by Dr. Dre and DJ Yella.', ' \"We Want Eazy\" also appears on his greatest hits, \"Eternal E\"; a 12-inch remix of this song was released as a single in 1989 and appeared on the rapper\\'s posthumous compilation, \"Featuring...Eazy-E\".']], ['Tyga discography', ['The discography of Tyga, an American rapper, consists of four studio albums, two compilation albums, fourteen mixtapes, eight singles (including four as a featured artist) and forty-eight music videos.', ' In 2008, Tyga released his first studio album, \"No Introduction\", on the record label Decaydance Records.', ' The album featured the single \"Coconut Juice\", which features singer Travie McCoy; the song peaked at number 94 on the US \"Billboard\" Hot 100, becoming Tyga\\'s first song to appear on the chart.', ' In 2010, Tyga and American singer Chris Brown released the collaborative mixtape \"Fan of a Fan\", which included the single \"Deuces\": the song peaked at number 14 on the \"Billboard\" Hot 100 and became Tyga\\'s first song to chart on the US Hot R&B/Hip-Hop Songs chart, peaking at number thirty.', ' Tyga also appeared on the song \"Loyalty\", a single by fellow rapper Birdman, and collaborated with rapper Lil Wayne on the non-album single \"I\\'m on It\", both of which failed to chart on the \"Billboard\" Hot 100.']], [\"Ego Trippin' (Part Two)\", ['\"Ego Trippin\\' (Part Two)\" is a 1994 single by the group, De La Soul, and the second single to be released from the group\\'s 1993 album, \"Buhloone Mindstate\".', ' The song (and its music video) were a scathing parody on gangsta rap complete with \"hardcore\" screaming.', ' The video caught the attention of rappers such as Ice Cube and Tupac Shakur, the latter taking exception to the video showing a rapper splashing around in a pool similar to Shakur\\'s own video for his song entitled \"I Get Around\".', ' The song features vocal contributions from Philadelphia rapper Shorty No Mas who also appeared in the video.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.714\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae7a5f05542993210983ece', 'answer': 'fourth', 'question': 'This कसबा पेठ विधानसभा मतदारसंघ is an example of a language that ranks what in the number of native speakers in India?', 'supporting_facts': [['Kasba Peth (Vidhan Sabha constituency)', 0], ['Marathi language', 3]], 'context': [['Nagpur South (Vidhan Sabha constituency)', ['Nagpur South Vidhan Sabha constituency (Marathi: नागपूर दक्षिण विधानसभा मतदारसंघ ) is one of the 288 Vidhan Sabha (legislative assembly) constituencies of Maharashtra state, western India.', ' The Constituency Number is 53.', ' This constituency is located in the Nagpur district.', ' The delimitation of the constituency happened in 2008.', ' It comprises parts of Nagpur Taluka, and Ward No. 9 to 11, 37 to 42, 73 to 78, 99 to 102 and 120 of Nagpur Municipal Corporation.']], ['Madha (Vidhan Sabha constituency)', ['Madha Vidhan Sabha constituency (Marathi: मढा विधानसभा मतदारसंघ ) is one of the 288 Vidhan Sabha (legislative assembly) constituencies of Maharashtra state in western India.']], ['Nagpur Central (Vidhan Sabha constituency)', ['Nagpur Central (Vidhan Sabha constituency) (Marathi: नागपूर मध्य विधानसभा मतदारसंघ ) is one of the 288 Vidhan Sabha (legislative assembly) constituencies of Maharashtra state, western India.', ' The Constituency Number is 55.', ' This constituency is located in the Nagpur district.', ' The delimitation of the constituency happened in 2008.', ' It comprises parts of Nagpur Taluka, and Ward No. 66, 92 to 98, 109 to 119 and 121 to 129.', ' of Nagpur Municipal Corporation.']], ['Vikhroli (Vidhan Sabha constituency)', ['Vikhroli Vidhan Sabha constituency (Marathi: विक्रोळी विधानसभा मतदारसंघ ) is one of the 288 Vidhan Sabha (Legislative Assembly) constituencies of Maharashtra state in western India.']], ['Language revitalization', ['Language revitalization, also referred to as language revival or reversing language shift, is an attempt to halt or reverse the decline of a language or to revive an extinct one.', ' Those involved can include parties such as linguists, cultural or community groups, or governments.', ' Some argue for a distinction between language revival (the resurrection of a dead language with no existing native speakers) and language revitalization (the rescue of a \"dying\" language).', ' It has been pointed out that there has only been one successful instance of a complete language revival, that of the Hebrew language, creating a new generation of native speakers without any pre-existing native speakers as a model.']], ['Marathi language', ['Marathi ( ; मराठी \"Marāṭhī \"; ] ) is an Indian language spoken predominantly by the Marathi people of Maharashtra.', ' It is the official language and co-official language in the Maharashtra and Goa states of Western India, respectively, and is one of the 22 scheduled languages of India.', ' There were 73 million speakers in 2007; Marathi ranks 19th in the list of most spoken languages in the world.', ' Marathi has the fourth largest number of native speakers in India, after Hindi, Bengali and Telugu in that order.', ' Marathi has some of the oldest literature of all modern Indo-Aryan languages, dating from about 900 AD.', ' The major dialects of Marathi are Standard Marathi and the Varhadi dialect.', ' Malvani Konkani has been heavily influenced by Marathi varieties.', ' The earliest example of the existence of Marathi as an independent language dates back to more than 2,000 years']], ['Maval (Vidhan Sabha constituency)', ['Maval Vidhan Sabha constituency (Marathi: मावळ विधानसभा मतदारसंघ ) is one of the twenty one constituencies of Maharashtra Vidhan Sabha located in the Pune district, India.']], ['Kasba Peth (Vidhan Sabha constituency)', ['Kasba Peth Vidhan Sabha constituency (Marathi: कसबा पेठ विधानसभा मतदारसंघ ) is one of the 288 Vidhan Sabha (legislative assembly) constituencies of Maharashtra state in Western India.', ' This constituency is located in the Pune district']], ['Nagpur East (Vidhan Sabha constituency)', ['Nagpur East (Vidhan Sabha constituency) (Marathi: नागपूर पूर्व विधानसभा मतदारसंघ ) is one of the 288 Vidhan Sabha (legislative assembly) constituencies of Maharashtra state, western India.', ' The Constituency Number is 54.', ' This constituency is located in the Nagpur district.', ' The delimitation of the constituency happened in 2008.', ' It comprises part of Nagpur Taluka and Ward No. 6 to 8, 28 to 36, and 67 to 72 of Nagpur Municipal Corporation.']], ['Vandre East (Vidhan Sabha constituency)', ['Vandre East Vidhan Sabha constituency (Marathi: वांद्रे पूर्व विधानसभा मतदारसंघ ) is one of the 288 Vidhan Sabha constituencies of Maharashtra state in western India.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.715\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae60426554299546bf83019', 'answer': 'BraveStarr', 'question': 'Which American Space Western animated series did Celebrity Home Entertainment released ', 'supporting_facts': [['Celebrity Home Entertainment', 0], ['BraveStarr', 0]], 'context': [['The Fairies', ['The Fairies is a live action Australian television show based on two fairies: Harmony and Rainbow Rhapsody (known in later series as just Rhapsody) .', ' They also have friends, including Elf, the Fairycake maker, Barnaby, the Bizzy Buzzy Bee and Wizzy the Wizard.', ' Each episode would contain songs and dance routines.', ' Originally it was a direct to video Series released by Carlton Home Entertainment UK in 2000, before becoming a series in 2005.', \" Contender Home Entertainment released the later series on DVD in 2007, with merchandise releasing in Australian throughout the programme's run.\"]], ['Monster Force', ['Monster Force is a 13-episode animated television series created in 1994 by Universal Cartoon Studios and Canadian studio Lacewood Productions.', ' The story is set in approx. 2020 and centers on a group of teenagers who, with help of high tech weaponry, fight off against classic Universal Monsters and spiritual beings threatening humanity.', ' Some of the crew have personal vendettas (e.g., one has the \"\"curse of the Wolfman\"\" that has been handed down through generations and another had a family member taken away from her by Dracula), while others fight for Mankind out of a sense of altruism.', ' The series aired in syndication alongside another Universal animated series, \"Exosquad\".', ' Universal Studios Home Entertainment released the first seven episodes to DVD on September 15, 2009.']], ['Magna Home Entertainment', ['Magna Home Entertainment was an independent home entertainment distributor headquartered in Brisbane, Queensland, Australia, operating within Australia and New Zealand.', ' As of February 2009, Magna Home Entertainment became a fully owned subsidiary of Beyond International (ASX:BYI), an Australian television production and distribution company.', ' Magna Home Entertainment is the sister company of Melbourne-based home entertainment distributor Beyond Home Entertainment, also a subsidiary of Beyond International.', ' Magna Home Entertainment distributes television series, documentaries and feature films.']], ['Sterling Entertainment Group', ['Sterling Entertainment Group (formerly United American Video Corporation, and more commonly known as United American Video, UAV Corporation or UAV Entertainment), was an entertainment company founded in 1984 as a small local company originally located in Nashville, Tennessee, then Charlotte, North Carolina starting in 1991.', ' Its headquarters would later relocate to Fort Mill, South Carolina in 1996.', ' UAV was also the longtime competitor of GoodTimes Entertainment, Anchor Bay Entertainment and Celebrity Home Video and many other sell through home entertainment companies.']], ['Tottoi', ['Tottoi (トトイ ) is a 1992 anime film.', ' It was dubbed into English and was distributed by Celebrity Home Entertainment in 1993 under the title of \"The Secret of the Seal\"']], ['Noel C. Bloom', ['Noel Christopher Bloom Sr. (born November 5, 1942) is an American businessman from Los Angeles.', ' He is notable for founding the entertainment and home video companies Artisan Entertainment, Family Home Entertainment, Celebrity Home Entertainment, Live Entertainment, Caballero Home Video, and Monterey Home Video.', ' Three of those companies of which founded are now owned by Lionsgate.', ' Bloom is married and has a daughter, Nicole (born 1970) and a son, Noel Jr. (born 1977).']], ['Celebrity Home Entertainment', ['Celebrity Home Entertainment (also known as simply \"Celebrity Video\"), founded by Noel C. Bloom in 1985, was a home video distributor specializing in mostly obscure material from around the world, as well as B-grade action films and soft-core adult fare, although they also released some material that was very famous at the time of its original release (such as \"BraveStarr\", \"Filmation\\'s Ghostbusters\", \"C.O.P.S.\" and the ).']], ['BraveStarr', ['BraveStarr is a 1980s American Space Western animated series.', ' The original episodes aired from September 1987 to February 1988 in syndication.', ' It was created simultaneously with a collection of action figures.', ' \"BraveStarr\" was the last animated series produced by Filmation and Group W Productions to be broadcast before Filmation shut down in 1989.', ' \"Bravo!\"', ', a spin-off series (originally called \"Quest of the Prairie People\") was in production along with \"Bugzburg\" when the studio closed down.', ' Reruns of the show aired on Qubo Night Owl from 2010 to 2013, and on the Retro Television Network from 2010 to 2015.']], ['COPS (animated TV series)', ['COPS (Central Organization of Police Specialists) is an American animated television series released by DIC Entertainment (distributed by Claster Television) and Celebrity Home Entertainment (some VHS tapes went through Golden Book Video, though).', ' This cartoon, which ran from 1988 to 1989, used the tag line: \"Fighting crime in a future time, protecting Empire City from Big Boss and his gang of crooks\".', ' In 1993, the series was shown in reruns on CBS Saturday mornings as \"CyberCOPS\", the name change due to the 1989 debut of the unrelated primetime reality show of the same name.', \" The show was based on Hasbro's 1988 line of action figures called C.O.P.S 'N' Crooks.\"]], ['Locke the Superman', ['Locke the Superman (超人ロック , Chōjin Rokku ) is a manga series by Yuki Hijiri which was later adapted into a movie and three OVA releases.', ' The movie was given an obscure video release in the United States by Celebrity Home Entertainment as \"Locke the Superpower\" which was rather heavily edited to 92 minutes, removing violence, nudity and any adult bits.', ' Both it and the OVAs were later licensed and released by Central Park Media under the original name.', ' Ten volumes were published in Poland under the title \"Locke Superczłowiek\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.716\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8133725542995ce29dcbdb', 'answer': 'Robert Erskine Childers DSC', 'question': 'Which writer was from England, Henry Roth or Robert Erskine Childers?', 'supporting_facts': [['Henry Roth', 0], ['Robert Erskine Childers', 0]], 'context': [['Asgard (yacht)', ['Asgard is a 51 ft gaff rigged yacht.', ' She was owned by the English-born writer and Irish nationalist Erskine Childers and his wife Molly Childers.', ' She is most noted for her use in the Howth gun-running of 1914.']], ['Henry Roth', ['Henry Roth (February 8, 1906 – October 13, 1995) was an American novelist and short story writer.']], ['R509 road (Ireland)', ['The R509 road, following part of the Childers Road (named after Erskine Childers), is a regional road in Ireland, running through the southeastern side of Limerick City.', ' It forms what is somewhat akin to an inner ring road (albeit mostly two-lane only).']], ['Molly Childers', ['Mary Alden Osgood Childers, MBE (14 December 1875 – 1 January 1964) was an American-born Irish writer and Irish nationalist.', ' She was the daughter of Dr Hamilton Osgood and Margaret Cushing Osgood of Beacon Hill, Boston, Massachusetts.', ' Her older sister was Gretchen Osgood Warren.', ' Molly married the writer and Irish nationalist, Robert Erskine Childers.', ' Their son, Erskine Hamilton Childers, became the fourth President of Ireland.']], ['Gretchen Osgood Warren', ['Gretchen Osgood Warren (19 March 1868 – September 1961), the wife of Fiske Warren, was an actress, singer and poet.', ' The daughter of Dr. Hamilton Osgood and Margaret Cushing Osgood of Beacon Hill, Boston, Massachusetts, her younger sister was Mary Alden Childers, the wife of writer and Irish nationalist Robert Erskine Childers.', ' Her nephew Erskine Hamilton Childers served as the fourth President of Ireland from 1973-74.']], ['Robert Caesar Childers', ['Robert Caesar Childers (1838 – 25 July 1876) was a British Orientalist scholar, compiler of the first Pāli-English dictionary.', ' Childers was the husband of Anna Barton of Ireland.', ' He was the father of Irish nationalist Robert Erskine Childers and grandfather to the fourth President of Ireland, Erskine Hamilton Childers.']], ['Robert Erskine Childers', ['Robert Erskine Childers DSC (25 June 1870 – 24 November 1922), universally known as Erskine Childers, was a British writer, whose works included the influential novel \"The Riddle of the Sands\", and a Fenian revolutionary who smuggled guns to Ireland in his sailing yacht \"Asgard\".', ' He was executed by the authorities of the nascent Irish Free State during the Irish Civil War.', ' He was the son of British Orientalist scholar Robert Caesar Childers; the cousin of Hugh Childers and Robert Barton; and the father of the fourth President of Ireland, Erskine Hamilton Childers.']], ['Irish Bulletin', ['The Irish Bulletin was the official gazette of the government of the Irish Republic.', ' It was produced by the Department of Propaganda during the Irish War of Independence.', ' and its offices were originally located at No. 6 Harcourt Street, Dublin.', \" The paper's first editor was Desmond FitzGerald, until his arrest and replacement by Robert Erskine Childers.\", ' \"The Bulletin\" appeared in weekly editions from 11 November 1919 to 11 July 1921.']], ['Erskine Barton Childers', ['Erskine Barton Childers (11 March 1929 – 25 August 1996) was an Irish writer, BBC correspondent and United Nations senior civil servant.', \" He was the eldest son of Erskine Hamilton Childers (Ireland's fourth President) and Ruth Ellen Dow Childers.\", \" His grandparents Mary Alden Childers and Robert Erskine Childers and the latter's double first cousin Robert Barton were all Irish nationalists involved heavily with the negotiation of Irish independence; which ultimately led to his grandfather's execution during the Irish Civil War.\", ' His great aunt was Gretchen Osgood Warren.']], ['Robert Barton', ['Robert Childers Barton (4 March 1881 – 10 August 1975) was an Irish nationalist, politician and farmer who participated in the negotiations leading up to the signature of the Anglo-Irish Treaty.', ' His father was Charles William Barton and his mother was Agnes Childers.', ' His wife was Rachel Warren of Boston, daughter of Fiske Warren.', ' His double first cousin and close friend was Robert Erskine Childers.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.717\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a83d7535542993344746093', 'answer': 'Los Angeles Xtreme, San Francisco Demons and Memphis Maniax', 'question': 'What other teams played in the same division that Chad Clements played in?', 'supporting_facts': [['Chuck Clements', 4], ['Las Vegas Outlaws (XFL)', 0], ['Las Vegas Outlaws (XFL)', 1]], 'context': [['Bertram Clements', ['Bertram Arthur Clements (1 December 1913 – July 2000) was an English footballer who represented Great Britain at the 1936 Summer Olympics.', ' Clements played amateur football for Casuals.']], ['1999–2000 BAI Basket', ['The 1999–2000 Season of BAI Basket (31st edition) ran from November 20, 2008 through May 16, 2000, with 8 teams playing in three different stages: in stage one (regular season) teams played a double round robin system.', ' In stage two, the six best teams played a single round robin tournament in serie A and the last six did the same for the consolation group, serie B. Finally, in stage three (final four) the best four teams from serie A played in a round robin at four rounds for the title.', ' The winners of the regular season and of the serie A are awarded a bonus point for the serie A and the final four, respectively.']], ['2007–08 BAI Basket', ['The 2007-2008 Season of BAI Basket (30th edition) ran from November 21, 2008 through May 16, 2009, with 12 teams playing in three different stages: in stage one (regular season) teams played a double round robin system.', ' In stage two, the six best teams played a single round robin tournament in serie A and the last six did the same for the consolation group, serie B. Finally, in stage three (final four) the best four teams from serie A played in a round robin at four rounds for the title.', ' The winners of the regular season and of the serie A are awarded a bonus point for the serie A and the final four, respectively.']], ['2009–10 BAI Basket', ['The 2009-2010 Season of BAI Basket (32nd edition) ran from November 13, 2009 to June 15, 2010, with 12 teams playing in three different stages: in stage one (regular season) teams played a double round robin system.', ' In stage two, the six best teams played a single round robin tournament in serie A and the last six did the same for the consolation group, serie B. Finally, in stage three (final four) the best four teams from serie A played in a round robin at four rounds for the title.', ' The winners of the regular season and of the serie A are awarded a bonus point for the serie A and the final four, respectively.']], ['Las Vegas Outlaws (XFL)', ['The Las Vegas Outlaws were an American football team in the XFL.', ' They played in the Western Division with the Los Angeles Xtreme, San Francisco Demons and Memphis Maniax.', ' They played their home games at Sam Boyd Stadium.', ' The Outlaws hosted the first nationally televised XFL game on NBC against the New York/New Jersey Hitmen.']], ['Al-Minaa SC–Naft Al-Janoob SC rivalry', ['Southern Iraqi football clubs Al-Minaa and Naft Al-Janoob have been rivals since the 2004–05 season when Naft Al-Janoob club started playing in the Premier League.', ' The clubs are respectively from Al-Maqal and Al-Tamimia, in the same city Basra, and for this reason a match between the two teams is sometimes called a \"Basra Derby\".', ' Another name is often used in the press is \"South Derby\", which comes from the location of Basra province in southern Iraq.', ' The animosity intensified since the first match, as Naft Al-Janoob was not expected to win Al-Minaa 1–0, and the exaggerated protest by Al-Minaa supporters to referee of match Khalil Yousuf prompted him to retire arbitration forever.', ' and this animosity reached a peak during the 2010–11 season, when both teams played at the end of the season in the Premier League in a match, that if it end at a draw, Naft Al-Janoob will relegate to the Iraq Division One.', ' Indeed, the match ended in a draw, and Al-Minaa fans celebrated the relegation of Naft Al-Janoob, and considered it a winning of league title.', ' In the 2015–16 season, Naft Al-Janoob returned to avenge Al-Minaa, when both teams played at the end of the season in the Premier League.', ' Al-Minaa needed two goals to go to the final, but Naft Al-Janoob played a defensive squad until the end of the match, although they were losing 1–0.']], ['2008–09 BAI Basket', ['The 2008-2009 Season of BAI Basket (31st edition) ran from November 20, 2008 through May 16, 2009, with 12 teams playing in three different stages: in stage one (regular season) teams played a double round robin system.', ' In stage two, the six best teams played a single round robin tournament in serie A and the last six did the same for the consolation group, serie B. Finally, in stage three (final four) the best four teams from serie A played in a round robin at four rounds for the title.', ' The winners of the regular season and of the serie A are awarded a bonus point for the serie A and the final four, respectively.']], ['Newport News Dodgers', ['The Newport News Dodgers were a minor league baseball affiliate of the Brooklyn Dodgers between 1944 and 1955.', ' They played in the Piedmont League and were based in Newport News, Virginia.', ' Gil Hodges played for this team in 1946.', ' Previously, Newport News teams were the Newport News Builders (1942), Newport News Pilots (1941), Newport News Shipbuilders (1900-1901; 1911-1922).', ' The teams played at Peninsula War Memorial Stadium on Pembroke Avenue in Hampton, Virginia.', ' The stadium was build by Brooklyn Dodgers President Branch Rickey.', ' The Dodgers played there from 1948-1955.', \" Previously, Newport News teams played at Builders' Park on Warwick Road (1944-1947) and prior to that at a ballpark on Wickham Avenue on the East End of Newport News.\", \" The Dodgers' move to Los Angeles in 1955 caused the team to realign its minor league affiliations, ending Newport News' franchise.\"]], ['List of KHL vs NHL games', ['Although the NHL teams played against Soviet league teams during the Super Series between 1976 and 1991, there were no games between post-Soviet and NHL teams until 2008, when Metallurg Magnitogorsk played against the New York Rangers for the 2008 Victoria Cup.', ' Two years later, in 2010, marked the first time since 1990 that NHL teams played games on post-Soviet ice.']], ['Chuck Clements', ['Chad \"Chuck\" Clements (born September 29, 1973) is a former American football quarterback who played one season with the New York Jets of the National Football League (NFL).', ' He was drafted by the New York Jets in the sixth round of the 1997 NFL Draft.', ' He played college football at the University of Houston and attended Huntsville High School in Huntsville, Texas.', ' He was also a member of the Philadelphia Eagles, Denver Broncos, Berlin Thunder, Las Vegas Outlaws and Ottawa Renegades.', ' Clements was drafted fifth overall by the Las Vegas Outlaws in the 2001 XFL Draft but, because of a preseason injury, never played for them.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.718\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae63cc55542992ae0d16298', 'answer': 'General Sam Houston', 'question': 'Who led the Texian Army in the battle in whichJoseph L. Bennett served as lieutenant colonel?', 'supporting_facts': [['Joseph L. Bennett', 1], ['Battle of San Jacinto', 1]], 'context': [['Joseph L. Bennett', ['Joseph L. Bennett (fl.', ' 1830s, died October 20, 1848) was an early settler of Montgomery County, Texas who served as lieutenant colonel in the Battle of San Jacinto and the Somervell Expedition.', ' He later served in the Texas House of Representatives from 1838-1840.']], ['Lieutenant colonel', ['Lieutenant colonel is a rank of commissioned officer in the armies and most marine forces and some air forces of the world, above a major and below a colonel.', ' The rank of lieutenant colonel is often shortened to simply \"colonel\" in conversation and in unofficial correspondence.', \" Sometimes, the term, 'half-colonel' is used in casual conversation in the British Army.\", ' A lieutenant colonel is typically in charge of a battalion in the army.']], ['Battle of San Jacinto', ['The Battle of San Jacinto, fought on April 21, 1836, in present-day Harris County, Texas, was the decisive battle of the Texas Revolution.', \" Led by General Sam Houston, the Texian Army engaged and defeated General Antonio López de Santa Anna's Mexican army in a fight that lasted just 18 minutes.\"]], ['Texian Army', ['The Texian Army, also known as the Army of the People, was a military organization consisting of volunteer and regular soldiers who fought against the Mexican army during the Texas Revolution.', ' Approximately 3,700 men joined the army between October 2, 1835, during the Battle of Gonzales through the end of the war on April 21, 1836, at the Battle of San Jacinto.', ' After gaining independence the Texian Army would be officially known as the Army of the Republic of Texas.', ' In 1846, after the annexation of Texas by the United States, the Army of the Republic of Texas merged with the US Army.', ' Sam Houston became the new commander in chief of the new Texas army.']], ['Grass Fight', ['The Grass Fight was a small battle during the Texas Revolution, fought between the Mexican Army and the Texian Army.', ' The battle took place on November 26, 1835, just south of San Antonio de Béxar in the Mexican region of Texas.', ' The Texas Revolution had officially begun on October 2 and by the end of the month the Texian had initiated a siege of Béxar, home of the largest Mexican garrison in the province.', ' Bored with the inactivity, many of the Texian soldiers returned home; a smaller number of adventurers from the United States arrived to replace them.', \" After the Texian Army rejected commander-in-chief Stephen F. Austin's call to launch an assault on Béxar on November 22, Austin resigned from the army.\", ' The men elected Edward Burleson their new commander-in-chief.']], ['Battle of Coleto', ['The Battle of Coleto, also known as the Battle of Coleto Creek, the Battle of the Prairie, and the Batalla del encinal del Perdido, was fought on March 19–20, 1836, during the Goliad campaign of the Texas Revolution.', ' In February, General José de Urrea led a branch of the Mexican army up the Gulf Coast of Mexican Texas toward Goliad, where a large contingent of soldiers from the Texian Army were garrisoned under Colonel James W. Fannin.', ' Simultaneously, Mexican president Antonio Lopez de Santa Anna led a larger force into the Texian interior, where on March 6 his troops won the Battle of the Alamo.', \" After learning of the Alamo's defeat, Texian general Sam Houston ordered Fannin to retreat from Goliad and join the rest of the army in Victoria.\"]], ['Siege of Béxar', ['The Siege of Béxar (or Bejar) was an early campaign of the Texas Revolution in which a volunteer Texian army defeated Mexican forces at San Antonio de Béxar (now San Antonio, Texas, US).', \" Texians had become disillusioned with the Mexican government as President and General Antonio Lopez de Santa Anna's tenure became increasingly dictatorial.\", ' In early October, 1835, Texas settlers gathered in Gonzales to stop Mexican troops from reclaiming a small cannon.', ' The resulting skirmish, known as the Battle of Gonzales, launched the Texas Revolution.', ' Men continued to assemble in Gonzales and soon established the Texian Army.', ' Despite a lack of military training, well-respected local leader General Stephen F. Austin was elected commander.']], ['Battle of Bennington', ['The Battle of Bennington was a battle of the American Revolutionary War, part of the Saratoga campaign, that took place on August 16, 1777, in Walloomsac, New York, about 10 mi from its namesake Bennington, Vermont.', \" A rebel force of 2,000 men, primarily composed of New Hampshire and Massachusetts militiamen, led by General John Stark, and reinforced by Vermont militiamen led by Colonel Seth Warner and members of the Green Mountain Boys, decisively defeated a detachment of General John Burgoyne's army led by Lieutenant Colonel Friedrich Baum, and supported by additional men under Lieutenant Colonel Heinrich von Breymann.\"]], ['Qazi Altaf Hussain', ['Lieutenant Colonel Qazi Altaf Hussain (1920–1999) served in the British Indian Army later taking up a place in the Army of Pakistan.', ' He advanced to various positions of leadership during his military career, serving as lieutenant colonel of the 11 Frontier Force Regiment, commandant of the Zhob Militia in Quetta, Pakistan, and commander of a regiment in the Indo-Pakistani War of 1965.', ' He was forced to retire as a lieutenant colonel, instead of advancing to general, as a result of his short-tempered, frank and outspoken nature.']], ['1st Para-Commando Battalion', ['The 1st Para-Commando Battalion (also known as The Cheetahs) is an elite Commando unit in the Bangladesh Army, established in 1976 inspired by the British Special Air Service.', ' Headquartered in Sylhet, its raising commanding officer was Lieutenant Colonel Md.', ' Zahurul Alam (retired as brigadier general in 2010).', ' Some renowned officers of this unit are (before raising this unit, they were commandos but they were not in any commando unit.', ' But now the commando unit has been raised so that their name has been kept in this unit.', \" They all are war heroes, so it's also an honour for this unit): Late Lieutenant General Ziaur Rahman, Late Brigadier Khaled Mosharraf, Late Colonel A.T.M. Haider, Late Colonel Abu Taher, Lieutenant Colonel (Shaheed) Abdus Salam, Lieutenant Colonel Md.\", ' Mustafizur Rahman, Colonel Saiful Islam, Late Major M. Anwar Hossain (Hell Commando) etc.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.719\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab7f3285542992aa3b8c88f', 'answer': 'the Haitian Revolution', 'question': 'Suzanne Simone Baptiste Louverture is the wife of the leader of which revolution?', 'supporting_facts': [['Suzanne Simone Baptiste Louverture', 0], ['Toussaint Louverture', 0]], 'context': [['Suzanne Simone Baptiste Louverture', ['Suzanne Simone Baptiste Louverture (around 1742 – May 19, 1816 in Agen, France) was the wife of Toussaint Louverture and the \"Dame-Consort\" of the French colony of Saint-Domingue.']], ['War of Knives', ['The War of Knives (French: \"Guerre des couteaux\"), also known as the War of the South, was a civil war from June 1799 to July 1800 between the Haitian revolutionary Toussaint Louverture, a black ex-slave who controlled the north of Saint-Domingue (modern-day Haiti), and his adversary André Rigaud, a free colored person of mixed race who controlled the south.', ' Louverture and Rigaud fought over de facto control of the French colony of Saint-Domingue during the war, which took place after the two men had successfully expelled foreign forces from the colony as part of the Haitian Revolution.', ' The war resulted in Toussaint taking control of the entirety of Saint-Domingue, and Rigaud fleeing into exile.']], ['Jean Baptiste Brunet', ['Jean Baptiste Brunet (7 July 1763 – 21 September 1824) was a French general of division in the French Revolutionary Army.', ' He was responsible for the arrest of Toussaint Louverture.', ' He was promoted to command a light infantry demi-brigade at the Fleurus in 1794.', \" He led the unit in François Joseph Lefebvre's division in the 1795, 1796 and 1799 campaigns.\", ' He was the son of French general Gaspard Jean-Baptiste Brunet who was guillotined in 1793.']], [\"Toussaint L'Overture County Cemetery\", [\"The Toussaint L'Ouverture County Cemetery is an historical African-American cemetery located in Franklin, Tennessee.\", ' It was listed on the National Register of Historic Places in 1995.', ' It is named for Toussaint Louverture, the leader of the Haitian Revolution.', \" The earliest recorded burials date from 1869, but it wasn't officially incorporated until 1884.\", ' It is \"the oldest African American institution in continuous use\" in Williamson County.']], ['Sanité Bélair', ['Suzanne Bélair, called \"Sanité Bélair\", (1781 – 5 October 1802), was a Haitian Freedom fighter and revolutionary, lieutenant in the army of Toussaint Louverture.']], ['Toussaint Louverture', [\"François-Dominique Toussaint Louverture (] 20 May 1743 – 7 April 1803), also known as Toussaint L'Ouverture or Toussaint Bréda, was the best-known leader of the Haitian Revolution.\", ' His military and political acumen saved the gains of the first Black insurrection in November 1791.', ' He first fought for the Spanish against the French; then for France against Spain and Great Britain; and finally, for Saint-Domingue against Napoleonic France.', ' He then helped transform the insurgency into a revolutionary movement, which by 1800 had turned Saint-Domingue, the most prosperous slave colony of the time, into the first free colonial society to have explicitly rejected race as the basis of social ranking.']], ['Revolution (Nina Simone song)', ['\"Revolution\" is a 1969 song by American jazz musician Nina Simone and Weldon Irvine.', ' It was released as a single in 1969 and on the album \"To Love Somebody\" in 1969.', ' The single release was split over two sides of a 45 rpm disc and these two edits were used as separate tracks on the album.', ' The song was released the year after the Beatles\\' \"Revolution\", and is seen by some as a variation of that song.', ' \"Revolution\" didn\\'t do as well as expected and Simone has expressed surprise and disappointment at its lack of success.']], ['Simone Torres', ['Simone Torres arrives in the fictional town of Pine Valley, Pennsylvania in 2001 at the behest of Edmund Grey in order to aid his brother-in-law Mateo Santos uncover the identity of the drug lord \"Proteus\".', \" An investigative reporter, Simone works to find Proteus's true identity while pretending to be engaged in an affair with Mateo, in order to protect the latter's wife and son from Proteus's wrath.\", ' Simone falls in love with Mateo, but it is clear his heart lies with his wife Hayley.', ' Simone mistakenly believes that a legitimate book publisher is interested in her authoring a book on the investigation.', ' Revealing inside information to the publisher, Simone is shocked when it turns out to be a front set up by \"Vanessa Cortlandt\", the real Proteus.', ' As a result of Simone\\'s leak, FBI agent \"Chris Stamp\" is almost killed.', ' A furious Mateo distances himself from Simone.']], ['United States and the Haitian Revolution', ['The Haitian Revolution provoked mixed reactions in the United States.', ' In June 1793 when the Haitian people, led by Toussaint Louverture, overthrew the French colonial rule and declared themselves an independent colony, it made the neighboring United States uneasy.', ' The slaves in Saint-Domingue’s were able to observe the growing disunity among the white colonists and themselves.', ' They realized that they would need to seek an opportunity to stop the tyranny that was being placed upon them, thus they took a stand and revolted.', ' The Haitian Revolution of 1804 impacted the United States of America, led by Thomas Jefferson, instilling fear of racial instability in the US, and the possible problematic effect the revolution could have on the early foreign relations and trade between the US and the new independent Haiti.', ' Thomas Jefferson realized that the Haitian Revolution had the potential to cause an upheaval against slavery in the US not only by the slaves themselves, but by white abolitionists as well.', ' Southern slaveholders feared that the revolt might spread from the island of Hispaniola to the slave plantations of the Southern United States.', ' The primary goal of the US was to maintain social order in the country, so the United States attempted to suppress the Haitian Revolution.', ' The US even went as far as to refuse acknowledgement of Haitian independence until 1862, which was during the heat of the North American civil war; coincidentally the main causal factor for the war between the states was slavery.', ' The second major impact that the Haitian Revolution had on the United States was on early foreign relations and trade that had been conducted with Haiti.', ' The United States had conducted trade and commerce with the Haitian island under French rule during the eighteenth century.', ' Haiti was the main producer of the United States supply of sugar and coffee, and once the Haitian slave population had broken from slavery, the US was reluctant to continue trade with them in fear that they would upset the French and the Southern slaveholders.', ' American merchants conducted a substantial trade with the plantations on Hispaniola (aka the French colony of Saint Domingue or Haiti).', ' But there were anti-slavery advocates in northern cities who believed that consistency with the principles of the American Revolution — life, liberty and equality for all—demanded that the U.S. support the slave insurgents.', ' An extremely beneficial aspect and real estate triumph that resulted from the Haitian Revolution and impacted the United States was the Louisiana Purchase.', ' Once Napoleon had lost his control of the land holding in the Caribbean to the Haitian rebellion, he felt that the French territory in the southern part of the United States was useless to the French Empire.', ' The US was only interested in the New Orleans area; however, the revolution enabled the sale of the entire territory west of the Mississippi River for around $15 million.', ' This purchase more than doubled the United States’ territory.']], ['Joseph Saint-Rémy', ['Joseph Saint-Rémy (1818 - 1856) was a Haitian historian.', ' He is best known for his biography \"La Vie de Toussaint Louverture\" about the Haitian Revolution leader Toussaint L\\'Ouverture, and for his work \"Pétion et Haïti\", about another Revolutionary figure, Alexandre Pétion.', ' Born in Guadeloupe, Saint-Rémy emigrated to Haiti as a young child and grew up in Les Cayes before leaving for school in France.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.720\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ade949b5542997c77adee63', 'answer': '1982', 'question': 'The city Charles Prince Airport is approximately 16 km northwest of was called Salisbury until what year?', 'supporting_facts': [['Charles Prince Airport', 0], ['Harare', 0]], 'context': [['Kosowa Niwa', ['Kosowa Niwa is a village in the administrative district of Gmina Czersk, within Chojnice County, Pomeranian Voivodeship, in northern Poland.', ' It lies approximately 16 km south-west of Czersk, 16 km east of Chojnice, and 92 km south-west of the regional capital Gdańsk.']], ['RAF El Amiriya', ['RAF El Amiriya is a former Royal Air Force military airfield in Egypt, located approximately 16\\xa0km south-southwest of Alexandria; 180\\xa0km northwest of Cairo']], ['Two and One-Half Mile Village', ['Two and One-Half Mile Village is an Indian settlement in southeast Yukon, Canada.', ' It is located on the Robert Campbell Highway (Highway 4), approximately 16 km northwest of Watson Lake.', ' The settlement is recognized as a census subdivision by Statistics Canada.']], ['Osieki Słupskie', ['Osieki Słupskie (German \"Wusseken\", Kreis \"Stolp\") is a village in the administrative district of Gmina Ustka, within Słupsk County, Pomeranian Voivodeship, in northern Poland.', ' It lies approximately 16 km east of Ustka, 16 km north of Słupsk, and 104 km west of the regional capital Gdańsk.']], ['Gąbino', ['Gąbino , (German: \"Gambin\" ) is a village in the administrative district of Gmina Ustka, within Słupsk County, Pomeranian Voivodeship, in northern Poland.', ' It lies approximately 16 km east of Ustka, 16 km north of Słupsk, and 103 km west of the regional capital Gdańsk.']], ['Pniewo, Gryfice County', ['Pniewo (German: \"Pinnow\" ) is a village in the administrative district of Gmina Płoty, within Gryfice County, West Pomeranian Voivodeship, in north-western Poland.', ' It lies approximately 16 km north-east of Płoty, 16 km east of Gryfice, and 78 km north-east of the regional capital Szczecin.']], ['Charles Prince Airport', ['Charles Prince Airport (ICAO: FVCP) , formerly named Mount Hampden and renamed after former airport manager Charles Prince (who was a Royal Air Force officer during World War II), is approximately 16 km northwest of Harare, Zimbabwe.']], ['Konigort', ['Konigort is a settlement in the administrative district of Gmina Czersk, within Chojnice County, Pomeranian Voivodeship, in northern Poland.', ' It lies approximately 16 km west of Czersk, 16 km north-east of Chojnice, and 89 km south-west of the regional capital Gdańsk.']], ['Harare', ['Harare ( ; officially called Salisbury until 1982) is the capital and most populous city of Zimbabwe.', ' Situated in the north-east of the country in the heart of historic Mashonaland, the city has an estimated population of 1,606,000 (2009), with 2,800,000 in its metropolitan area (2006).', ' Administratively, Harare is a metropolitan province, which also incorporates Chitungwiza town and Epworth.', ' It is situated at an elevation of 1,483 m above sea level and its climate falls into the subtropical highland category.']], ['Zielony Kąt', ['Zielony Kąt is a village in the administrative district of Gmina Nowodwór, within Ryki County, Lublin Voivodeship, in eastern Poland.', ' It lies approximately 16 km east of Ryki and 55 km northwest of the regional capital Lublin.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 278/500 [00:00<00:00, 1345.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.720\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8b057255429971feec4611', 'answer': 'Wolf Creek', 'question': \"David Lightfoot worked on what film loosely based on the murders of tourists by Ivan Milat in the 1990's?\", 'supporting_facts': [['David Lightfoot', 0], ['Wolf Creek (film)', 2]], 'context': [['Mark Tedeschi', ['Mark Alfred Guido Tedeschi, AM, QC (born 1952) is an Australian barrister, law professor, photographer and author.', ' He has won numerous awards for his photography and has been featured in galleries throughout the world including in the State Library of New South Wales, the New South Wales Art Gallery, the Center for Fine Art Photography in Colorado, and the National Library in Canberra.', ' He is the Senior Crown Prosecutor for New South Wales and the Head of Chambers of the 84 Crown Prosecutors.', ' He is the founder and president of the Australian Association of Crown Prosecutors and a visiting professor at the University of Wollongong.', ' As a prosecutor, Tedeschi is best known for the prosecution of numerous high-profile cases in Australia including the 2006 conviction of Dr. Suman Sood for illegal abortion and the Backpacker Murders committed by Ivan Milat in the 1990s.']], ['Coffin Rock', ['Coffin Rock is an Australian melodramatic thriller film directed by Rupert Glasson and produced by David Lightfoot.', ' The movie stars Lisa Chappell, Robert Taylor and Sam Parsonson.']], ['Backpacker murders', ['The backpacker murders were a spate of serial killings that took place in New South Wales, Australia, between 1989 and 1993, committed by Ivan Milat.', ' The bodies of seven missing young people aged 19 to 22 were discovered partially buried in the Belanglo State Forest, 15 km south west of the New South Wales town of Berrima.', ' Five of the victims were foreign backpackers visiting Australia (three German, two British), and two were Australian travellers from Melbourne.']], ['David Lightfoot', ['David Lightfoot worked on the horror films \"Wolf Creek\" and \"Rogue\".']], ['Wing Commander (film)', ['Wing Commander is a 1999 science fiction film loosely based on the video game series of the same name.', ' It was directed by Chris Roberts, the creator of the game series, and stars Freddie Prinze, Jr., Matthew Lillard, Saffron Burrows, Tchéky Karyo, Jürgen Prochnow, David Suchet, and David Warner.']], ['Wanted (2008 film)', ['Wanted is a 2008 American-German action thriller film loosely based on the comic book miniseries of the same name by Mark Millar and J. G. Jones.', ' The film, written by Chris Morgan, Michael Brandt, and Derek Haas and directed by Timur Bekmambetov stars James McAvoy, Morgan Freeman, and Angelina Jolie.', ' The storyline follows Wesley Gibson (McAvoy), a frustrated account manager who discovers that he is the son of a professional assassin and decides to join the Fraternity, a secret society in which his father worked.']], ['Rock-a-Doodle', ['Rock-a-Doodle is a 1991 live action/animated musical comedy film loosely based on Edmond Rostand\\'s comedy \"Chantecler\".', ' Directed by Don Bluth and written by David N. Weiss, \"Rock-a-Doodle\" is an Irish, British and American venture produced by Sullivan Bluth Studios and Goldcrest Films.', ' The film features the voices of Glen Campbell, Christopher Plummer, Phil Harris (in his final role before his retirement and death), Charles Nelson Reilly, Sorrell Booke, Sandy Duncan, Eddie Deezen, Ellen Greene and Toby Scott Ganger in his film debut.', ' The film was released in the United Kingdom on 2 August 1991, and in the United States on 3 April 1992.']], ['Wolf Creek (film)', ['Wolf Creek is a 2005 Australian horror film written, co-produced, and directed by Greg McLean, and starring John Jarratt.', ' The story revolves around three backpackers who find themselves taken captive and after a brief escape, hunted down by Mick Taylor in the Australian outback.', ' The film was ambiguously marketed as being \"based on true events\"; the plot bore elements reminiscent of the real-life murders of tourists by Ivan Milat in the 1990s and Bradley Murdoch in 2001.']], ['Catching Milat', ['Catching Milat is a two-part Australian television miniseries that screened on the Seven Network, in collaboration with \"Screen Australia\" on 17 and 24 May 2015.', ' It is based on the book \"Sins of the Brother\" by Mark Whittaker and Les Kennedy and is loosely based upon the true story of how NSW Police and detectives under \"Task Force Air\" tracked down and caught serial killer Ivan Milat, who was responsible for the infamous backpacker murders.']], ['The Long Island Serial Killer (film)', ['The Long Island Serial Killer (also known as The Gilgo Beach Murders) is a 2013 American true crime horror film loosely based on the elusive Long Island serial killer who murdered seventeen women on Long Island between 1996 and 2010.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.721\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab1d7ac554299449642c7e6', 'answer': 'Ehretia', 'question': 'Which genus has more species, Xanthoceras or Ehretia?', 'supporting_facts': [['Xanthoceras', 0], ['Ehretia', 0], ['Ehretia', 1]], 'context': [['Xanthoceras', ['Xanthoceras sorbifolium (yellowhorn, shiny leaf yellowhorn, goldenhorn, Chinese flowering chestnut) is a woody perennial in the soapberry family, Sapindaceae, and the only species in the genus Xanthoceras.', ' It is native to northern China in the provinces of Gansu, Hebei, Henan, Liaoning, Nei Monggol, Ningxia, Shaanxi, and Shandong.', ' It is also cultivated in Russia, having been imported there since the 19th Century.']], ['Ehretia', ['Ehretia is a genus of flowering plants in the borage family, Boraginaceae.', ' It contains about 50 species.', ' The generic name honors German botanical illustrator Georg Dionysius Ehret (1708–1770).']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.722\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7481d755429974ef308c12', 'answer': 'CBS', 'question': 'On which television network did Joe Penny play an investigator named Jake Styles?', 'supporting_facts': [['Alan Campbell (actor)', 0], ['Jake and the Fatman', 0]], 'context': [['Jake and the Fatman', ['Jake and the Fatman is a television crime drama starring William Conrad as prosecutor J. L. (Jason Lochinvar) \"Fatman\" McCabe and Joe Penny as investigator Jake Styles.']], ['Across the River to Motor City', ['Across the River to Motor City is a Canadian television drama series, that aired on City stations.', ' It debuted November 22, 2007.', ' The series is about an insurance investigator named Ben Ford who works the border in both Detroit and Windsor.', ' The story takes into account the shifting allegiances and ambitions that straddle the Detroit/Windsor boundary, an urban portion of the Canada/United States border.']], ['Flying Jake', [\"Flying Jake is a children's picture book by Lane Smith.\", ' It was originally published in 1988 by Macmillan Publishing Company and reprinted by Viking Press in 1996.', ' In the wordless story, a boy named Jake takes flight in pursuit of his pet bird, which has flown out of its cage and through a window.', ' \"Flying Jake\" was the first independent work by Smith, who later illustrated \"The True Story of the 3 Little Pigs!', '\" and \"The Stinky Cheese Man\".']], ['Justin Case (film)', ['Justin Case is a 1988 television film by Blake Edwards.', ' George Carlin stars as a private investigator named Justin Case.', ' Justin is found dead in his office by Jennifer Spalding (Molly Hagan) who is an out of work dancer there for an interview for a secretary/receptionist position.', ' Justin comes back as a ghost that only Jennifer can see, and convinces her to help unravel the mystery of his murder.']], ['Breach of Faith: A Family of Cops 2', ['Breach of Faith: A Family of Cops 2 is a 1997 action television film starring Charles Bronson sequel to \"A Family of Cops\".', ' In the film, Joe Penny takes over the role of eldest son Ben Fein, who was played by Daniel Baldwin in the first film.', ' This was the second-to-last film Charles starred in before his death in 2003.']], ['Vinnie Fiorello', ['Vinnie Fiorello (born June 24, 1974) is an American drummer, lyricist and a founding member of the ska punk band Less Than Jake.', ' As a child, Fiorello\\'s family owned a dog named Jake who was \"treated like a king\" according to the band\\'s website FAQ page.', ' As a result, everything in the house became \"Less Than Jake\", spawning the band\\'s name.']], ['Alan Campbell (actor)', ['Bruce Alan Campbell (born April 22, 1957), credited professionally as Alan Campbell, is an American television, film and stage actor who is best known for his roles as Derek Mitchell in the 1987–1992 CBS series \"Jake and the Fatman\" and as E.Z. Taylor on the short-lived 1984–1985 \"Three\\'s Company\" spin-off \"Three\\'s a Crowd\".']], ['Joe Penny', ['Joseph Edward Penny Jr. (born 24 June 1956) is an English-born American actor best known for his roles as Nick Ryder on the detective series \"Riptide\" from 1984–86, and as Jake Styles in the CBS television series \"Jake and the Fatman\" from 1987–92.']], ['Joe Pennington', ['Joe Pennington, aka \"Joe Penny,\" (born January 15, 1928 in Plant City, Florida) is a former lead guitarist for Hank Williams\\' backing band, the Drifting Cowboys.', ' After leaving the Drifting Cowboys in 1948, Pennington continued to perform and recorded several pioneering rockabilly singles on the Federal Records label in the mid-1950s.']], ['The Gangster Chronicles', ['The Gangster Chronicles is an NBC American television crime drama miniseries starring Michael Nouri, Joe Penny, Jon Polito, Louis Giambalvo, Kathleen Lloyd, Madeleine Stowe, Chad Redding, Markie Post, Allan Arbus, James Andronica, Robert Davi, Joseph Mascolo, and narrated by E.G. Marshall.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.722\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac1b8cd55429963665198fc', 'answer': 'Brothers Quay', 'question': 'Who won more awards, Brothers Quay or Jake Kasdan?', 'supporting_facts': [['Brothers Quay', 0], ['Brothers Quay', 2], ['Jake Kasdan', 0]], 'context': [['Pilot (New Girl)', ['\"Pilot\" is the first episode of the first season of \"New Girl\".', ' The episode was written by Elizabeth Meriwether, and directed by Jake Kasdan.', ' The episode first aired on Fox in the United States on September 20, 2011 to positive reviews.']], ['Sex Tape (film)', ['Sex Tape is a 2014 American comedy film directed by Jake Kasdan and written by Kate Angelo, Jason Segel, and Nicholas Stoller.', ' Starring Segel, Cameron Diaz, Rob Corddry, Ellie Kemper, and Rob Lowe, the film was released on July 18, 2014, by Columbia Pictures.']], ['Jake Kasdan', ['Jacob \"Jake\" Kasdan (born October 28, 1974) is an American television and film director and occasional actor.']], ['Walk Hard: The Dewey Cox Story', ['Walk Hard: The Dewey Cox Story is a 2007 American comedy film written and produced by Judd Apatow and Jake Kasdan, directed by Kasdan and starring John C. Reilly.', ' The plot echoes the storyline of 2005\\'s Johnny Cash biopic \"Walk the Line\" and 2004\\'s Ray Charles biopic \"Ray\"; \"Walk Hard\" is also a parody of the biopic genre as a whole.']], ['The Piano Tuner of Earthquakes', ['The Piano Tuner of Earthquakes is a 2005 film by the Brothers Quay, featuring Amira Casar, Gottfried John and Assumpta Serna.', ' It was the second feature-length film by the Brothers Quay and their first film in over ten years.']], ['Brothers Quay', ['Stephen and Timothy Quay ( ; born June 17, 1947) are American identical twin brothers better known as the Brothers Quay or Quay Brothers.', ' They are influential stop-motion animators.', ' They are also the recipients of the 1998 Drama Desk Award for Outstanding Set Design for their work on the play \"The Chairs\".']], ['Bad Teacher', ['Bad Teacher is a 2011 American comedy film directed by Jake Kasdan based on a screenplay by Lee Eisenberg and Gene Stupnitsky, and starring Cameron Diaz, Justin Timberlake, Lucy Punch, Jason Segel, and Phyllis Smith.']], ['Speechless (TV series)', ['Speechless is an American sitcom television series that debuted on ABC on September 21, 2016.', ' Created by Scott Silveri and co-executive produced with Christine Gernon, Jake Kasdan and Melvin Mar, the 20th Century Fox Television/ABC Studios co-production was greenlighted to series order on May 13, 2016.', ' A first-look trailer was released on the same day.', ' On September 29, 2016, the series was picked up for a full 22-episode season.', ' An additional episode was ordered on December 13, 2016, for a 23-episode season.']], ['Weird Loners', ['Weird Loners is an American comedy television series created by Michael J. Weithorn.', ' The 6-episode first season was ordered straight-to-series by the Fox network in 2014.', ' The series is executive produced by Weithorn and Jake Kasdan.', ' The series premiered on March 31, 2015.']], ['Ben and Kate', ['Ben and Kate is an American single-camera sitcom television series that ran on Fox from September 25, 2012, to January 22, 2013, as part of the 2012–13 television season.', ' The show was produced by 20th Century Fox Television and Chernin Entertainment.', ' The show was created by Dana Fox who served as an executive producer alongside Peter Chernin, Katherie Pope, and Jake Kasdan.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.723\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a81e8a0554299676cceb157', 'answer': 'Goodbye Mr Mackenzie', 'question': 'Which band has more members, Chvrches or Goodbye Mr. Mackenzie?', 'supporting_facts': [['Chvrches', 1], ['Goodbye Mr. Mackenzie', 0], ['Goodbye Mr. Mackenzie', 1]], 'context': [['Martin Metcalfe', ['Martin Metcalfe is a singer and artist from Edinburgh, Scotland.', ' He previously played in Goodbye Mr. Mackenzie, Angelfish and Isa & the Filthy Tongues.', ' He is currently in a band called The Fornicators, and also paints.']], ['Chvrches', ['Chvrches (pronounced and written as \"churches\" in Roman form and stylised as CHVRCHΞS) are a Scottish synth-pop band from Glasgow, formed in September 2011.', ' The band consists of Lauren Mayberry (lead vocals, drums, additional synthesisers, samplers), Iain Cook (synthesisers, guitar, bass, vocals) and Martin Doherty (synthesisers, samplers, vocals).']], ['Goodbye Mr. Mackenzie', ['Goodbye Mr Mackenzie (known informally as \"The Mackenzies\") was a Scottish 1980s and 1990s rock group formed in Bathgate, near Edinburgh, Scotland.', \" At the band's commercial peak, the line-up consisted of Martin Metcalfe on vocals, John Duncan on guitar, Fin Wilson on bass guitar, Shirley Manson and Rona Scobie on keyboards and backing vocals, and Derek Kelly on drums.\"]], [\"Jacob's Ladder (album)\", [\"Jacob's Ladder is the first album from the Scottish group The Filthy Tongues and was self-released in 2016 by the band on their own label Blokshok Records, in association with Neon Tetra.\", ' The album follows \"Addiction\" and \"Dark Passenger\", two records recorded by an earlier incarnation of the band known as Isa & the Filthy Tongues, as well as following the albums released by the members earlier major label bands Goodbye Mr. Mackenzie and Angelfish.']], ['Angelfish (album)', ['Angelfish is the 1994 self-titled debut and only studio album released by Scottish alternative rock group Angelfish, fronted by Shirley Manson.', ' The \"Angelfish\" album was born out of necessity when Goodbye Mr. Mackenzie\\'s record distributor MCA expressed interest in recording an album with Manson on lead vocals rather than furthering its commitment to the Mackenzies.', \" The Mackenzies' record label boss Gary Kurfirst signed Manson as a solo artist, with the remaining Mackenzies performing as her backing band to circumvent the band's existing deal with MCA.\"]], ['Blacker Than Black', ['\"Blacker Than Black\" is a 1990 single by Scottish alternative rock group Goodbye Mr. Mackenzie.', ' \"Blacker Than Black\" was the bands second and final single release under the Parlophone label, and preceded its parent album \"Hammer and Tongs\" by almost a year.', ' In 1991, after Goodbye Mr. Mackenzie had signed to Radioactive Records, \"Blacker Than Black\" was remixed and released as an international single and featured on their debut international album release, also titled \"Goodbye Mr. Mackenzie\".']], ['Shirley Manson discography', ['This is a discography of Scottish recording artist Shirley Manson, who has performed as the lead singer of American rock band shes pretty good but shes no David Bowie.', ' Garbage since 1994.', ' Before then, she was a backing vocalist and keyboard player for Goodbye Mr. Mackenzie from 1981 to 1992.', ' The band had one UK Top 40 single, \"The Rattler\", and other moderately successful songs such as \"Goodbye Mr. Mackenzie\".', ' Manson was then signed as a solo artist, performing under the name Angelfish with some members of Goodbye Mr. Mackenzie, releasing \"Angelfish\" in the early 1990s.']], ['Hammer and Tongs', ['Hammer and Tongs is the second album from the Scottish rock group Goodbye Mr. Mackenzie.', \" It was recorded in Germany in 1989, at Berlin's Hansa Ton Studios just as the Fall of the Berlin Wall occurred.\", ' The album sat on the shelf for almost 18 months, in which time the band were transferred across EMI record labels, from Capitol to Parlophone, who released two singles from the album in 1990.', ' Parlophone sold the band\\'s record deal to Radioactive Records and MCA, who released \"Hammer and Tongs\" in the United Kingdom in early 1991 and encouraged the band to record a new song \"Now We Are Married\" to promote the release.']], ['Goodbye Mr. Mackenzie (song)', ['Goodbye Mr. Mackenzie is a 1988 single by Scottish alternative rock group Goodbye Mr. Mackenzie.', \" It was their debut major label single, recorded for EMI's Capitol Records, following a string of well-received independent releases.\", ' Supported by the band performing the single live on national music show \"The Tube\", \"Goodbye Mr. Mackenzie\" was the band\\'s first single to reach the Top 75 of the UK Singles chart, when it peaked at #62 on its third week of release.']], ['Five (Goodbye Mr Mackenzie album)', ['Five is the third album from the Scottish group Goodbye Mr. Mackenzie and was self-released in 1994 by the band on their own label Blokshok Records.', ' It has a rougher edge to it than their two previous major label albums, partly due to Martin Metcalfe being inspired by grunge music and The Pixies.', ' The album was supported by the extended play release of \"Hard\" as a single.', ' \"Five\" failed to revive interest outside of the band’s existing fanbase.', ' Metcalfe later explained: \"We turned to a harder sound and radio wasn’t prepared to play it\".']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.724\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adf39d55542992d7e9f92e6', 'answer': 'Romantic', 'question': 'Heinrich Marschner was a composer who performed in the time frame after one of the first significant composers in what school of work?', 'supporting_facts': [['Heinrich Marschner', 0], ['Carl Maria von Weber', 0]], 'context': [['Tule fog', [\"Tule fog is a thick ground fog that settles in the San Joaquin Valley and Sacramento Valley areas of California's Great Central Valley.\", \" Tule fog forms from late fall through early spring (California's rainy season) after the first significant rainfall.\", ' The official time frame for tule fog to form is from November 1 to March 31.', ' This phenomenon is named after the tule grass wetlands (\"tulares\") of the Central Valley.', ' Tule fog is the leading cause of weather-related accidents in California.']], ['List of operas by Marschner', ['This is a complete list of the operas of the German composer Heinrich Marschner (1795–1861).']], ['Heinrich Marschner', ['Heinrich August Marschner (16 August 1795 – 14 December 1861) was the most important composer of German opera between Weber and Wagner.']], ['Seymour Shifrin', ['Seymour Shifrin (28 February 1926 – 26 September 1979) was an American composer.', ' He was described by \"Time Magazine\" as \"one of the most significant composers of his generation.\"']], ['Carl Maria von Weber', ['Carl Maria Friedrich Ernst von Weber (18 or 19 November 1786 5 June 1826) was a German composer, conductor, pianist, guitarist and critic, and was one of the first significant composers of the Romantic school.']], ['Marschnerstraße', ['The Marschnerstraße, named after the composer Heinrich Marschner (1795-1861), is a street founded in 1897, in the Munich district of Pasing and Obermenzing.']], ['The Vampyr: A Soap Opera', ['The Vampyr: A Soap Opera is a miniseries based on Heinrich Marschner\\'s opera \"Der Vampyr\".', ' It first aired on BBC 2 on December 2, 1992.', ' The new English libretto was written by Charles Hart, based on a story by Janet Street-Porter and Nigel Finch, which was based on the original libretto by Wilhelm August Wohlbrück, which was based on John Polidori\\'s short story \"The Vampyre.\"', ' It was conducted by David Parry and directed by Nigel Finch.']], ['George Whitefield Chadwick', ['George Whitefield Chadwick (November 13, 1854 – April 4, 1931) was an American composer.', ' Along with Horatio Parker, Amy Beach, Arthur Foote, and Edward MacDowell, he was a representative composer of what is called the Second New England School of American composers of the late 19th century—the generation before Charles Ives.', \" Chadwick's works are influenced by the Realist movement in the arts, characterized by a down-to-earth depiction of people's lives.\", ' Many consider his music to portray a distinctively American style.', ' His works included several operas, three symphonies, five string quartets, tone poems, incidental music, songs and choral anthems.', ' Along with a group of other composers collectively known as the Boston Six, Chadwick was one of those responsible for the first significant body of concert music by composers from the United States.', ' The other five were Amy Beach, Arthur Foote, Edward MacDowell, John Knowles Paine, and Horatio Parker.']], ['Hans Heiling', [\"Hans Heiling is a German Romantic opera in 3 acts with prologue by Heinrich Marschner with a libretto by Eduard Devrient, who also sang the title role at the première at the Königliche Hofoper (now Berlin State Opera), Berlin, on 24 May 1833, and went on to become Marschner's most successful opera.\", ' The opera brought the composer a considerable reputation, although this did not materially affect his position in Hanover, where he was music director of the Court Theatre.', ' Like Marschner\\'s other great success, \"Der Vampyr\", the plot of \"Hans Heiling\" makes great use of supernatural elements.', ' As with several of his operas, \"Hans Heiling\" is based on a folk legend.']], ['Memory of Mankind', ['Several motivations underlie the project.', ' The primary ambition of MOM is to preserve an image of our era, created by numerous participants all over the planet.', ' MOM will also contain information which our society is obliged to forward to the future: e.g. description of nuclear waste repositories.', ' MOM collaborates with the NEA and SKB.', ' Although the most obvious ambition and often described in the media is the concern about preserving our knowledge, this is not the primary goal of MOM.', ' Serving as a time capsule MOM is both: in a time frame of millennia it is the story about us, and in a time frame of decades it is a backup.', ' In times where global warming, nuclear danger and biological warfare threaten the existence of civilization, saving the core knowledge and culture acquired over centuries is a backup measure.', ' In case of a collapse, the MOM project could help survivors to rebuild civilization.', ' Linked to this, an another reason is of political order: facing the lack of reactivity of authorities concerning global warming, the MOM project is a reminder of what can happen.', ' Roman and Greek civilizations whose histories have been reconstructed by the small percentage of texts and artifacts which survived until our days are examples that have inspired the MOM project.', \" Finally, it's a critic of our digital civilization : according to Kunze, maybe nothing of the 21st century will last in the future, since most of our interactions are now virtual.\", ' The \"accuracy versus bullshit\" is one of the main themes of the MOM project, worried about loss of information, the project can only aim to save a fragment of the information produced until today, but this fragment has to be representative.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.725\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abf08985542997719eab625', 'answer': 'Magic Dirt', 'question': 'Which band was formed first, Circus Diablo or Magic Dirt?', 'supporting_facts': [['Circus Diablo', 0], ['Magic Dirt', 0]], 'context': [['Girl (Magic Dirt album)', [\"'Girl' is the sixth album from Australian rock-grunge band Magic Dirt.\", ' Produced by Magic Dirt and Lindsay Gravina, at Birdland Studios, Melbourne.']], ['Billy Morrison', ['Billy Morrison is an English guitarist, singer and actor who plays guitar with Billy Idol and performs with the Los Angeles-based cover band Royal Machines (and previously with Camp Freddy).', ' Morrison previously fronted the hard rock act Circus Diablo and has been a member of The Cult, Stimulator, Doheny and Into a Circle.']], ['Beast (Magic Dirt album)', [\"'Beast' is a mini-album by from Australian rock-grunge band Magic Dirt.\", ' Produced by Magic Dirt and Lindsay Gravina, at Birdland Studios, Melbourne.']], ['Magic Dirt', ['Magic Dirt are an Australian rock band, which formed in 1991 in Geelong, Victoria, with Daniel Herring on guitar, Adam Robertson on drums, Adalita Srsen on vocals and guitar, and Dean Turner on bass guitar.', ' Initially forming an alternative underground band called Deer Bubbles which split and formed into the much heavier, rock based group called The Jim Jims, they were renamed as Magic Dirt in 1992.', ' Their top 40 releases on the ARIA Albums Chart are \"Friends in Danger\" (1996), \"What Are Rockstars Doing Today\" (2000), \"Tough Love\" (2003) and \"Snow White\" (2005).', ' They have received nine ARIA Music Award nominations including four at the ARIA Music Awards of 1995 for \"Life Was Better\" – their second extended play.', ' Turner died in August 2009 of dermatofibrosarcoma protuberans (a soft tissue cancer).', ' s of 2011 , the band is on hiatus with no immediate plans to tour or record.']], ['Life Was Better', ['Life Was Better was the second EP release by Australian rock band Magic Dirt.', ' It was released in November 1994 on the Melbourne independent label Au Go Go Records.', \" This was the band's first recording after regrouping in the wake of a split in April 1994, brought about by singer/guitarist Adalita Srsen and bass player Dean Turner ending their relationship.\"]], ['Bored (band)', ['Bored (stylised as Bored!)', ' were an Australian punk rock band which formed in Geelong in 1987.', ' The original line-up was Grant Gardner on bass guitar, Adrian Hann on keyboards, Justin Munday on drums, John Nolan on guitar (ex-Behind the Magnolia Curtain) and Dave Thomas on guitar and vocals (ex-Bodies, Slaughter House).', ' In 1989 Gardner was replaced by Tim Hemensley (ex-Royal Flush, God).', ' Both Hemensley and Nolan left in 1991 to form Powder Monkeys.', ' Bored!', ' released four studio albums by 1993 and disbanded later that year.', ' Thomas briefly joined Magic Dirt and subsequently has enlisted various line-ups for reformed versions of Bored!', ' in 1998, 1999 and 2000.']], ['Magic Dirt (US Version)', ['Magic Dirt is an album by Australian alternative rock band Magic Dirt released in the U.S. on the Dirt label.', ' It compiles the band\\'s first two Australian-released EPs \"Signs of Satanic Youth\" and \"Life Was Better\" along with additional tracks.']], ['Circus Diablo', ['Circus Diablo is an American rock band, formed in early 2006 by Billy Morrison (vocals), Billy Duffy (lead guitar) and Ricky Warwick (rhythm guitar).', ' Fuel frontman Brett Scallions and Velvet Revolver drummer Matt Sorum subsequently joined the band on bass and drums, respectively.', ' To date, Circus Diablo have released one studio album, entitled \"Circus Diablo\".']], ['The Dirt Bike Kid', ['The Dirt Bike Kid is a 1985 film directed by Hoite Caston, produced by Julie Corman, starring Peter Billingsley and Stuart Pankin, about a boy who discovers a magic dirt bike that has a mind of its own.', ' Part of the story is inspired by \"Jack and the Beanstalk\".']], ['Raúl Sánchez (musician)', ['Raúl Sánchez is a Spanish-born Australian rock musician, best known as the lead guitarist in Magic Dirt.', ' Raul initially played in Melbourne based band Muffcake before joining Magic Dirt.', ' Raul currently plays in River Of Snakes, and Tex Perkins super group \"The Ape\".']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.725\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abd4c6d5542993062266c47', 'answer': 'Tianhe Stadium', 'question': 'What is the home stadium where Dai Xianrong predominantly played?', 'supporting_facts': [['Dai Xianrong', 1], ['Guangzhou Evergrande Taobao F.C.', 1]], 'context': [['C.D. Mafra', ['Clube Desportivo Mafra is a football club currently playing in the Campeonato de Portugal.', ' They are based in the town of Mafra and own Campo Doutor Mário Silveira stadium, but the games are played in Estádio Municipal de Mafra.', \" Founded in 1965 the club predominantly played within Portugal's regional leagues where they gradually worked their way up to until they won the Associação de Futebol de Lisboa Division 1 title in the 1991–92 league season and promotion to the national leagues.\"]], ['Tårnby Boldklub', ['Tårnby Boldklub (or Taarnby Boldklub) were a Danish football club based in Tårnby on Amager, which played their home games at Taarnby Stadium.', ' The club was formed on 21 April 1935 as Arbejdernes Idræts Klub Tårnby (AIK Tårnby) and became member of Dansk Arbejder Idrætsforbund (DAI) on 1 May 1935.', ' After nearly 10 years in DAI, it was decided at a general meeting on 16 January 1944 to change their membership to Københavns Boldspil-Union (KBU) and in this regard change their name to Tårnby Boldklub.', \" The club became an extraordinary member of KBU on 1 July 1944 and a full member in February 1945 and started their first season 1944–45 under the new football association by playing in a newly created 'C-række' consisting of 8 clubs.\", ' The club enjoyed rivalries against neighbouring clubs Kastrup Boldklub and in the early days Sundby Boldklub.', ' In 2002 the club together with Kastrup Boldklub became a part of the superstructure Amager United.', ' In 2006, Kastrup Boldklub, however withdrew from the collaboration and the superstructure was dissolved.', ' On 1 January 2009, the club merged with nabouring Amager Boldklub af 1970 (AB70), and the new association was named AB Tårnby.']], ['Guangzhou Evergrande Taobao F.C.', ['Guangzhou Evergrande Taobao Football Club is a professional Chinese football club that participates in the Chinese Super League under the license of the Chinese Football Association.', ' The team is based in Guangzhou, Guangdong, and their home stadium is the Tianhe Stadium which has a seating capacity of 58,500.', ' Their majority shareholders are the Evergrande Real Estate Group (56.71%) and the e-commerce company Alibaba Group (37.81%) while the rest of the shares are traded in the Chinese OTC system.']], ['Dai Xianrong', ['Dai Xianrong (; born February 7, 1982) is a former professional Chinese footballer who mainly played as a centre-back.', ' Throughout his career he predominantly played for Guangzhou F.C. where he won two second tier Chinese league one titles with them in 2007 and 2010 before retiring.', ' And he was the member of U-23 China National Football Team.']], ['Stewart McKimmie', ['Stewart McKimmie (born 27 October 1962) is a Scottish former professional footballer, who predominantly played for home town club Aberdeen.', ' He played in defence, primarily as a right-back, and also played for Dundee and Dundee United.', \" He now writes a weekly column in the Evening Express, as well as appearing as a pundit on Northsound 2's Friday Sport.\"]], ['1987 NFL season', ['The 1987 NFL season was the 68th regular season of the National Football League.', ' This season featured games predominantly played by replacement players as the National Football League Players Association (NFLPA) players were on strike from weeks four to six.', ' The season ended with Super Bowl XXII, with the Washington Redskins defeating the Denver Broncos 42–10 at Jack Murphy Stadium in San Diego.', ' The Broncos suffered their second consecutive Super Bowl defeat.']], ['Mississippi Veterans Memorial Stadium', ['Mississippi Veterans Memorial Stadium is an outdoor football stadium in Jackson, Mississippi, United States.', ' Veterans Memorial Stadium is the home field of the Jackson State Tigers football team.', ' The stadium was originally known as War Veterans Memorial Stadium then later as Hinds County War Memorial Stadium before finally being christened with its current moniker.', ' In the past it has served as an alternate home stadium for The University of Mississippi, Mississippi State University, and the University of Southern Mississippi.', ' From 1973 to 1990 the Egg Bowl was played there and from 1992 to 2013 it hosted the Mississippi High School Activities Association state championship football games.', ' In addition to college and high school games it has hosted several National Football League (NFL) preseason games.']], ['1957 European Cup Final', ['The 1957 European Cup Final was a football match which took place at the Santiago Bernabéu Stadium in Madrid, Spain, on 30 May 1957.', ' It was contested between Real Madrid of Spain and Fiorentina of Italy.', ' Real Madrid won 2–0 after goals from Alfredo Di Stéfano and Francisco Gento in the second half.', ' It was the first of four finals (also counting the Champions League era, followed by the 1965, 1984 and 2012 finals) where one of the teams played in its home stadium, and also the first final where the winning team played at their home stadium.']], ['Yunnan Hongta F.C.', ['Yunnan Hongta (Simplified Chinese: 云南红塔) was a football club who played in the Chinese Jia-A League who were founded by the Shenzhen Jinpeng Group in 1996 and named Shenzhen Jinpeng (Simplefied Chinese: 深圳金鹏).', ' The club predominantly played within the lower leagues until they were sold to the Yunnan Hongta Group, a tobacco producer who renamed the team Yunnan Hongta and moved the club to Kunming to play in the Tuodong Stadium.', ' Yunnan Hongta would win promotion to the Chinese Jia-A League in the 1999 league season where they remained until Chongqing Lifan bought then merged the teams in 2003.']], ['War Memorial Stadium (Arkansas)', ['War Memorial Stadium is a multi-purpose stadium in Little Rock, Arkansas.', ' The stadium is primarily used for American football and is the home stadium for the Arkansas Baptist Buffaloes, Catholic High School Rockets, Little Rock Rangers and the secondary home stadium for the University of Arkansas Razorbacks.', ' The Arkansas State University Red Wolves have in the past played a few games there and will hold a spring game there in 2016.', ' The stadium also hosts the Delta Classic, an annual football game between the University of Arkansas at Pine Bluff Golden Lions and the Grambling State Tigers, as well as hosting the Arkansas Activities Association high school championship game in all classification.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.726\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8b6ee255429950cd6afcfd', 'answer': 'Dracula', 'question': 'Whiich genus has more species, Dracula or Pistacia?', 'supporting_facts': [['Dracula (plant)', 0], ['Pistacia', 1]], 'context': [['Anacardiaceae', ['The Anacardiaceae, commonly known as the cashew family or sumac family, are a family of flowering plants, including about 83 genera with about 860 known species.', ' Members of the Anacardiaceae bear fruits that are drupes and in some cases produce urushiol, an irritant.', ' The Anacardiaceae include numerous genera, several of which are economically important, notably cashew (in the type genus \"Anacardium\"), mango, poison ivy, sumac, smoke tree, marula, yellow mombin, and cuachalalate.', ' The genus \"Pistacia\" (which includes the pistachio and mastic tree) is now included, but was previously placed in its own family, the Pistaciaceae.']], ['Commiphora gileadensis', ['Commiphora gileadensis, the Arabian balsam tree is a shrub species in the genus \"Commiphora\" growing in Saudi Arabia, Yemen, southern Oman, and in southeast Egypt where it may have been introduced.', ' Other common names for the plant include balm of Gilead and Mecca myrrh, but this is due to historical confusion between several plants and the historically important expensive perfumes and drugs obtained from them.', ' True balm of Gilead was very rare, and appears to have been produced from the unrelated tree \"Pistacia lentiscus\".']], ['Dracula (plant)', ['The orchid genus Dracula, abbreviated as Drac in horticultural trade, consists of 118 species native to Mexico, Central America, Colombia, Ecuador and Peru.', ' The name \"Dracula\" literally means \"little dragon\", an allusion to the mythical Count Dracula, a lead character in numerous vampire novels and films.', ' The name was applied to the orchid because of the blood-red color of several of the species, the strange aspect of the long spurs of the sepals.']], ['Akbesia', ['Akbesia is a genus of moths in the Sphingidae family, containing only one species, the Pistacia hawkmoth, Akbesia davidi, which is known from southern Turkey, northern Syria, northern Israel, western Jordan, south-eastern Turkey, north-eastern Iraq, south-eastern Georgia, northern Iran, eastern Afghanistan and Iranian Beluchistan.', ' It may also occur across Azerbaijan, the Alborz Mountains of northern Iran, the Zagros Mountains of western and southern Iran, and northern Afghanistan.', ' It often occurs in large numbers at certain sites in rocky, hilly areas supporting scattered trees and shrubs of \"Quercus\", \"Olea\", \"Ceratonia\" and \"Pistacia\".']], ['Terebinth', ['Two similar species of the plant genus \"Pistacia\":']], ['Pistacia khinjuk', ['Pistacia khinjuk is a species of plant in the \"Pistacia\" genus which grows in Iran, northern Iraq, southern Jordan and Turkey.', ' The tree grows up to 10 metres.']], ['Pistacia chinensis', ['Pistacia chinensis (English: Chinese pistache ; ) is a small to medium-sized tree in the genus \"Pistacia\" in the cashew family, Anacardiaceae, native to central and western China.', ' It is hardy, can withstand harsh conditions and poor quality soils, and grows up to 20 m.', ' The leaves are deciduous, alternate, pinnate, 20–25\\xa0cm long, with 10 or 12 leaflets, the terminal leaflet usually absent.', ' The flowers are produced in panicles 15–20\\xa0cm long at the ends of the branches; it is dioecious, with separate male and female plants.', ' The fruit is a small red drupe, turning blue when ripe, containing a single seed.', ' This species is planted as a street tree in temperate areas worldwide due to its attractive fruit and autumn foliage.']], ['Pistacia aethiopica', ['Pistacia aethiopica is an African and Arabia coast peninsula species of plant in the Anacardiaceae family.', ' It is a dioecious evergreen shrub or tree of the pistacio genus, growing up to 20 m tall, adapted to the dry environment.', ' It is found in Ethiopia, Kenya, Somalia, Tanzania, Uganda, and Yemen.']], ['Dracula wallisii', ['Dracula wallisii is a species of orchid belonging to the genus \"Dracula\".', ' The species is found at altitudes of 1600 to in Cordillera Central, Colombia.', ' It is a common species, with large flowers that are often highly variable in form.']], ['Pistacia', ['Pistacia is a genus of flowering plants in the cashew family, Anacardiaceae.', ' It contains 10 to 20 species that are native to Africa and Eurasia from the Canary Islands, all of Africa, and southern Europe, warm and semidesert areas across Asia, and North America from Mexico to warm and semidesert United States, such as Texas or California.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.727\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7263e05542997f82783998', 'answer': 'Bonkyll Castle', 'question': 'What medieval fortress is believed to refer to a chapel at the bottom of a hill?', 'supporting_facts': [['Buncle', 2], ['Bonkyll Castle', 0]], 'context': [['Prozor Fortress', ['Prozor Fortress (Croatian: \"Tvrđava Prozor\" or \"Gradina\" ) is a medieval fortress situated in the continental part of Split-Dalmatia County, in inland Dalmatia, just above the town of Vrlika in Croatia.', ' From its origin as a small stronghold built by the ancient Illyrian tribe Dalmatae, it developed into a fortress in the 15th century, during the reign of the Croatian and Bosnian feudal lord Hrvoje Vukčić Hrvatinić.']], ['Tønsberg Fortress', ['Tønsberg Fortress (\"Tunsberg festning\") was a medieval fortress, located in Tønsberg, Norway which was defended by the fortress for over 300 years.']], ['Fortress of Klis', ['The Klis Fortress (Croatian: \"Tvrđava Klis\" ) is a medieval fortress situated above a village bearing the same name, near the city of Split, in central Dalmatia, Croatia.', ' From its origin as a small stronghold built by the ancient Illyrian tribe Dalmatae, becoming a royal castle that was the seat of many Croatian kings, to its final development as a large fortress during the Ottoman wars in Europe, Klis Fortress has guarded the frontier, being lost and re-conquered several times throughout its more-than-two-thousand-year-long history.', ' Due to its location on a pass that separates the mountains Mosor and Kozjak, the fortress served as a major source of defense in Dalmatia, especially against the Ottoman advance, and has been a key crossroad between the Mediterranean belt and the Balkan rear.']], ['Glamoč Fortress', ['Glamoč fortress (Bosnian: \"Glamočka tvrđava\" ) is a medieval fortress located on the north slopes of Staretina mountain just above town of Glamoč.', ' The construction of the fortress started as early as 14th century.']], ['Nicopsis', ['Nicopsis, Nikopsis, or Nikopsia (Greek: Νικόψις ; Georgian: ნიკოფსი, ნიკოფსია ) was a medieval fortress and town on the northeastern Black Sea coast, somewhere between the towns of Tuapse, Russia, and New Athos, Abkhazia/Georgia.', ' It features in the medieval Greek and Georgian sources as a Byzantine outpost and then as the northwestern extreme of the Kingdom of Georgia.', ' A center of Christianity in the region known as Zichia, Nikopsis was at times a Byzantine bishopric and was believed to be a burial place of the apostle Simon the Canaanite.']], ['Prizren Fortress', ['Prizren Fortress (Serbian: Призренски град\"/Prizrenski grad\" ), also known as Kaljaja (Albanian: \"Каlаја\" , ) and Dušan\\'s Fortress (Душанов град\"/Dušanov grad\" ), is a medieval fortress in Prizren, Kosovo, which once served as the capital of the Serbian Empire.', ' It was built on a hill above Prizrenska Bistrica, around which the modern city developed.', ' The first fort, erected on this location by the Byzantines, was further expanded by Emperor Stefan Dušan (r. 1331–55).', ' The fort then came under the control of the Ottomans for four centuries.', ' It was declared a Monument of Culture of Exceptional Importance in 1948.']], [\"Asen's Fortress\", ['Asen\\'s Fortress (Bulgarian: Асенова крепост , \"Asenova krepost\"), identified by some researchers as Petrich (Петрич), is a medieval fortress in the Bulgarian Rhodope Mountains, 2 to south of the town of Asenovgrad, on a high rocky ridge on the left bank of the Asenitsa River.', \" Asen's Fortress is 279 m above sea level.\"]], ['Saranta Kolones', ['Saranta Kolones(Greek: Κάστρο Σαράντα Κολώνες, Forty Columns castle ) is a ruined medieval fortress inside the Paphos Archaeological Park and it is located just north of the harbour of Paphos, on the island of Cyprus.', ' It takes its name from the large number of granite columns that were found on the site and probably once formed part of the ancient agora.', ' The Byzantine castle is believed to have been built at the end of the 7th century AD to protect the port and the city of Nea Pafos from Arab raids and later remodeled by the Lusignans.', ' The Fortress had a three-metre thick wall with four huge corner towers and another four intermediary towers along the joining walls and moat surrounding the castle.', ' Access was across a wooden bridge spanning the moat.', ' The square courtyard measured 35 metres long by 35 metres wide, with a tower at each corner.', ' The main entrance was through a fifth, horseshoe-shaped tower on the east side.', ' Destroyed by an earthquake in 1222, the castle was subsequently abandoned.']], ['Buncle', ['Buncle is a Celtic derived surname originating from Bonkyll Castle, Berwickshire, Scotland.', ' It has many modern variations that are phonetically similar, for example: Bonkle, Bonkyll, Bonkill, Bonkylle, Bonkile, Bunkle, Bunkall and Bonckle.', \" The names' meaning is believed to refer to a chapel at the bottom of hill.\"]], ['Bonkyll Castle', ['Bonkyll Castle (also variously spelled Bonkyl, Boncle, Buncle, Bunkle or Bonkill) was a medieval fortress situated in the eastern Scottish Borders of which little remains.', ' The site is protected as a scheduled monument.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.728\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac2381355429951e9e684a9', 'answer': 'Pac-12 Conference', 'question': 'Which conference is the team for which Justin Bannan played college football currently a member of?', 'supporting_facts': [['Justin Bannan', 2], ['Colorado Buffaloes football', 1]], 'context': [['Colorado Buffaloes football', ['The Colorado Buffaloes football program represents the University of Colorado Boulder in college football at the NCAA Division I FBS level.', ' The team is currently a member of the Pac-12 Conference, having previously been a charter member of the Big 12 Conference.', ' Before joining the Big 12, they were members of the Big Eight Conference.', ' The CU football team has played at Folsom Field since 1924.', ' The Buffs all-time record is 694–493–36 (.583 winning percentage) prior to the Valero Alamo Bowl at the end of the 2016 season.', ' Colorado won a National Championship in 1990.', ' The football program is 23rd on the all-time win list and 30th in all-time winning percentage.', ' The football team also has the distinction of being the all-time NCAA leader in 4th down conversions .', ' They are one of two NCAA Division I teams to complete a 5th down conversion (the other being Cornell).', ' This was a result of a mistake by the officials and happened on a play displayed by chaincrew as the 4th down.']], ['Joe Ruetz', ['Joseph Hubert Ruetz (October 21, 1916 – January 2, 2003) was a professional football player in the All-America Football Conference (AAFC) for the Chicago Rockets in 1946 and 1948.', ' Prior to that he played college football while attending the University of Notre Dame.', ' He played guard for the Irish with the exception of playing one season at quarterback.', ' In 1938, he graduated from Notre Dame with cum laude honors.', ' Ruetz played in the 1938 College All-Star Game and his team upset Sammy Baugh and the Washington Redskins.', ' During World War II, he was a Navy physical education instructor and pilot.', \" During that time he played for the famed Saint Mary's Pre-Flight football team.\", ' He was named an All-Navy All-American by sportswriter Grantland Rice in 1942.', ' After the war, Ruetz studied at the University of Chicago and played two seasons with the Chicago Rockets of the All-America Football Conference.', \" In 1950, he then was an assistant and head coach at St. Mary's before joining Chuck Taylor's football staff at Stanford University and helping guide the team to the 1952 Rose Bowl.\", ' Ruetz later worked as a fund-raiser for Cardinal athletics, before succeeding Taylor as athletic director in 1972.', ' He then \"saved\" the East-West Shrine Game by convincing Stanford to allow it to be played at the school, where it remained until its move to Pacific Bell Park in 2001.', \" Joe also launched the head coaching career of Bill Walsh in 1977, when he hired him as Stanford's football coach.\", ' He also instituted a long football series with Notre Dame, before retiring in 1979.', ' He later served as a fund-raising consultant to the Psoriasis Research Institute in Palo Alto, California.']], ['Abe Mickal', ['Ibrahim Khalil \"Abe\" Mickal ( \\u20091912/1913 – September 20, 2001) was a Lebanese-American college football player and a doctor.', ' He played as a halfback for the LSU Tigers football team of Louisiana State University, where he was notable for his passing skills and play-making ability, which earned him the nickname \"Miracle Mickal\".', \" He was also the team's primary punter and placekicker.\", ' A three-time All-Southeastern Conference (SEC) selection, Mickal led LSU to an undefeated season in 1933 and a conference championship and Sugar Bowl in 1935.', ' In 1936, Mickal played quarterback for a college all-star team that was the first team of college players to defeat a professional team.', ' Although selected in the 1936 NFL Draft, he did not play professionally.', ' Mickal was a charter member of the LSU Athletic Hall of Fame in 1937 and was inducted into the College Football Hall of Fame in 1967.']], ['Justin Harper (American football)', ['Justin Harper (born February 24, 1985 in Catawba, North Carolina) is an American college football coach and former professional Canadian football wide receiver.', ' He is currently the Interim Head Coach at Virginia State.', ' He was most recently a member of the BC Lions of the CFL.', ' He was drafted by the Baltimore Ravens in the seventh round of the 2008 NFL Draft.', ' He played college football at Virginia Tech.']], ['Josh Wilcox', ['Joshua David Wilcox (born June 5, 1974) is a former American football tight end who played two seasons with the New Orleans Saints of the National Football League.', ' He played college football at the University of Oregon and attended Junction High School in Junction City, Oregon.', ' Wilcox was also a member of the Portland Forest Dragons of the Arena Football League, the Amsterdam Admirals of NFL Europe and the Los Angeles Xtreme of the XFL.', ' He won the Million Dollar Game in the XFL as a member of the Los Angeles Xtreme.', ' Wilcox is the son of Pro Football Hall of Famer Dave Wilcox and brother of college football coach Justin Wilcox.']], ['Carroll Fighting Saints football', ['The Carroll Fighting Saint football program represents Carroll College of Helena, Montana in college football.', ' The team competes in the Frontier Conference, which is affiliated with the National Association of Intercollegiate Athletics (NAIA).', ' The Carroll Fighting Saints football team began playing in 1920 and is one of the most successful programs in the NAIA division of college football.', ' The program has won six NAIA Football National Championships (2002, 2003, 2004, 2005, 2007, 2010) and 40 conference championships, 14 while a member of the Montana Collegiate Conference and 26 as a member of the Frontier Conference.', ' The team is currently coached by Mike Van Diest who in his 17th season at Carroll, has compiled a career record of 194– 36.', \" His winning percentage of .8145 is the third highest of any head coach with at least ten seasons of experience in college football history, behind only those of Mount Union's Larry Kehres and Notre Dame's Knute Rockne.\", ' The Carroll College Fighting Saints plays their home games on campus at Nelson Stadium.']], ['Roy Roundtree', ['Roy Randolph Roundtree (born March 7, 1989) is a former American football wide receiver and current assistant coach for the Indiana State Sycamores.', ' He was a 2013 preseason member of the Cincinnati Bengals and played college football for the Michigan Wolverines football team where he spent his redshirt senior season with the 2012 team.', ' In 2012, he was an All-Big Ten honorable mention selection.', ' He was a 2011 Fred Biletnikoff Award preseason watchlist honoree.', \" He was a Fred Biletnikoff Award preseason watchlist honoree in 2010, and set Michigan's single-game receiving record with nine catches for 246 yards against Illinois that November.\", \" Roundtree was the team's leading receiver in both the 2009 and 2010 seasons.\", ' He finished first in the Big Ten Conference in receiving yards in 2010 for Conference games, and was a second team All Conference selection.', ' While in high school, he was named the 2007 Ohio Division II Offensive Player of the Year.']], ['Justin Bannan', ['Justin Lewis Bannan (born April 18, 1979) is a former American football defensive tackle.', ' He was drafted by the Buffalo Bills in the fifth round of the 2002 NFL Draft.', ' He played college football at Colorado.']], ['Hogan Wharton', ['Robert Glen \"Hogan\" Wharton (December 13, 1935 – October 8, 2008) was an American football player.', ' He attended the University of Houston where he played college football at the tackle position for the Houston Cougars football team from 1956 to 1958.', ' He was named lineman of the year in the Missouri Valley Conference in 1957, and the following year he was selected by the American Football Coaches Association as a first-team tackle on its 1958 College Football All-America Team.', \" Wharton later played professional football in the newly formed American Football League, playing at the guard position for the Houston Oilers during the first four years of the club's existence from 1960 to 1963, including the 1960 Houston Oilers team that won the first AFL championship.\", ' He was cut by the Oilers in September 1964.']], [\"Charlie O'Rourke\", ['Charles C. \"Chuckin\\' Charlie\" O\\'Rourke (May 10, 1917 – April 14, 2000) was an American football player and coach.', ' He played college football as a quarterback at Boston College and professionally with Chicago Bears of the National Football League (NFL) and the Los Angeles Dons and Baltimore Colts of the All-America Football Conference (AAFC).', \" O'Rourke quarterbacked the Boston College Eagles football team to one of its most famous wins.\", \" His 24-yard run late in the fourth quarter gave the 1940 Eagles a 19–13 victory over Tennessee in the 1941 Sugar Bowl, staking BC's claim to a national championship.\", \" O'Rourke served as the head football coach at University of Massachusetts Amherst (UMass) from 1952 to 1959, compiling a record of 21–39–4.\", ' In 1972, he came the first Boston College player to be inducted into the College Football Hall of Fame.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.729\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7be5bd5542997c3ec972ba', 'answer': 'The Leonberger is a giant dog breed.', 'question': 'Which dog is considered a giant dog breed, the  Leonberger or the Basset Hound?', 'supporting_facts': [['Leonberger', 0], ['Basset Hound', 0]], 'context': [['Dog Tales (1958 film)', ['Dog Tales is a 1958 Warner Brothers animated cartoon which consists of a series of blackout gags involving dogs (e.g., one in which a doberman pinscher viciously pinches an overweight U.S. Army private identified as \"Doberman\" (a reference to, and caricature of, the character played by Maurice Gosfield on \"The Phil Silvers Show\"); and another in which the narrator can\\'t make up his mind whether the dog pictured is a pointer or a setter, and then finally shows a picture of a \"point-setter\").', ' A basset hound declares that she\\'s a TV star (a reference to Cleo the Dog, from the contemporary TV sitcom \"The People\\'s Choice\"), we learn the unusual breed of a Newfoundland puppy\\'s grandfather, and a great dane named \"Victor Barky\" plays the piano.', ' Reused animation from Chuck Jones\\' \"Often an Orphan\" (1949) and Friz Freleng\\'s \"Piker\\'s Peak\" (1957) is also seen here.', ' In the former case, Charlie Dog makes a cameo - his final appearance in a Warner Bros. cartoon as well as his only cartoon to not be directed by Chuck Jones.', ' This animated film features the voices of Mel Blanc, Robert C. Bruce and Julie Bennett, and was directed by Robert McKimson and written by Tedd Pierce.', ' It was released in theaters on July 26, 1958.']], ['Barnyard Dawg', ['(The) Barnyard Dawg (also known as George P. Dog) is a \"Looney Tunes\" character.', ' An adult anthropomorphic basset hound, he is the archenemy of Foghorn Leghorn.', ' He was created by Robert McKimson, who also created Foghorn, and was voiced by Mel Blanc.', ' Dawg also feuds with other enemies as well like Henery Hawk, the Weasel, Daffy Duck and Sylvester.', ' He appeared in 22 Golden Age–era Warner Bros. shorts.']], ['Cão de Gado Transmontano', ['The Cão de Gado Transmontano (] ; English: Transmontano Mastiff or Transmontano Cattle Dog) is a rare molosser working giant dog breed, originating in and largely limited to the region of Trás-os-Montes e Alto Douro Province, Portugal.', ' Their primary function is flock and herd protection, and their success is interlinked to the welfare of the flock and the presence of wolves, particularly.', ' Although they are a regional breed, their wolf defense capacity has led to limited experimental importation elsewhere.']], ['Bulldog', ['The Bulldog is a medium-sized breed of dog commonly referred to as the English Bulldog or British Bulldog.', ' Other scent-hound breeds include the Small Greek Domestic Dog, Irish Wolfhound, Bluetick Coonhound, Finnish Lapphund, and the Basset Hound.', ' The Bulldog is a muscular, hefty dog with a wrinkled face and a distinctive pushed-in nose.', ' The American Kennel Club (AKC), The Kennel Club (UK), and the United Kennel Club (UKC) oversee breeding records.', ' Bulldogs were the fourth most popular purebreed in the US in 2007 according to the American Kennel Club.']], ['Fred Basset', ['Fred Basset is a comic strip about a male basset hound.', ' The cartoon was created by Scottish cartoonist Alex Graham and published first in the \"Daily Mail\" on 8 July 1963.', ' It has since been syndicated around the world.']], ['Leonberger', ['The Leonberger is a giant dog breed.', \" The breed's name derives from the city of Leonberg in Baden-Württemberg, Germany.\", ' According to legend, the Leonberger was ostensibly bred as a \"symbolic dog\" that would mimic the lion in the town crest.', ' It is in the Working Group for dog shows such as Crufts, but not at the World Dog Show.']], ['Basset Hound', ['The Basset Hound is a short-legged breed of dog of the hound family.', ' The Basset is a scent hound that was originally bred for the purpose of hunting hare.', ' Their sense of smell and ability to ground-scent is second only to that of the Bloodhound.', ' Basset Hounds are one of 6 recognised basset-type breeds in France.']], ['Breed standard (dogs)', ['A breed standard (also called bench standard or the standard) in the dog fancy is a set of guidelines covering specific \"externally observable\" qualities such as \"appearance\", \"movement\", and \"temperament\" for that dog breed.', ' Breed standards are not scientific documents, but are written for each breed by clubs of hobbyists called breed clubs for their own specific requirements.', ' Details and definitions within breed standards for a specific dog breed may vary from breed club to breed club and from country to country.', ' Dog breed standards are similar in form and function to breed standards for other domesticated animals.']], ['Giant dog breed', ['A giant dog breed has no universal height or weight classification, although some groups define \"giant breeds\" as those heavier than 50 kg .', ' Giant breeds grow rapidly, but take longer to mature into their full adult sizes than smaller dogs.']], ['List of U.S. state dogs', ['Twelve states of the United States have designated an official state dog breed.', ' Maryland was the first state to name a dog breed as a state symbol, naming the Chesapeake Bay Retriever in 1964.', ' Pennsylvania followed the year after, naming the Great Dane as its official breed.', ' Dog breeds are mostly affiliated with the states that they originated in.', ' North Carolina chose the Plott Hound as it was the only dog breed indigenous to the state.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.729\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a796bfd55429970f5fffeac', 'answer': 'A simple iron boar crest', 'question': 'What adorns the archaeological artefact excavated by Thomas Bateman on 3 May 1848?', 'supporting_facts': [['Pioneer Helmet', 2], ['Benty Grange helmet', 0]], 'context': [['May Assembly', ['May Assembly (Serbian: Мајска скупштина / Majska skupština ) was the national assembly of the Serbs in Austrian Empire, held on 1 and 3 May 1848 in Sremski Karlovci, during which the Serbs proclaimed autonomous Serbian Vojvodina.', ' This action was later recognized by the supreme Austrian authority in Vienna.', ' May Assembly was part of the European Revolutions of 1848.']], ['Artognou stone', ['The Artognou stone, sometimes referred to as the Arthur stone, is an archaeological artefact uncovered in Cornwall in the United Kingdom.', ' It was discovered in 1998 in securely dated sixth-century contexts among the ruins at Tintagel Castle in Cornwall, a secular, high status settlement of sub-Roman Britain.', ' It appears to have originally been a practice dedication stone for some building or other public structure, but it was broken in two and re-used as part of a drain when the original structure was destroyed.', ' Upon its discovery the stone achieved some notoriety due to the suggestion that \"Artognou\" was connected to the legendary King Arthur, though scholars such as John Koch have criticized the evidence for this connection.']], ['Slatino furnace model', ['The Slatino furnace model is an ancient ceramic artefact excavated at an archeological site near Slatino in Bulgaria.', ' It was found among the remains of a burned down dwelling dated from the Eneolithic period (ca. 5000 BCE).', \" The description 'furnace model' (and name) has been adopted in the absence of a definite idea about the use and meaning of the object.\", ' On its largest flat side there is a clearly traced rough']], ['Five Wells', ['Five Wells is a chambered tomb and scheduled ancient monument on Taddington Moor in the Peak District.', ' Three stones mark the main chamber, which has been dramatically reduced; a second less well-preserved chamber is to the west.', ' Access can be had on foot via a permitted path from Pillwell Gate to the west.', ' The mound was excavated by Thomas Bateman in 1846.']], ['Pioneer Helmet', ['The Pioneer Helmet (also known as Wollaston Helmet or Northamptonshire Helmet) is a 7th-century Anglo-Saxon boar-crested helm found by archaeologists from Northamptonshire Archaeology at a quarry site operated by Pioneer Aggregates.', ' This helmet is very similar in its basic design to the Coppergate Helmet, although it is much larger, and was likely to have had two cheek plates (of which only one remained) and a nasal (which was bent inwards at the time of deposition to render the piece unwearable).', ' A simple iron boar crest adorns the top of this helmet associating it with the Benty Grange helmet and the Guilden Morden boar from the same period, and descriptions in the poem Beowulf.', ' The helmet accompanied the burial of a young male, possibly laid on a bed with a pattern welded sword, small knife, hanging bowl, three iron buckles and a copper alloy clothes hook.']], ['Benty Grange helmet', ['The Benty Grange helmet is an archaeological artefact excavated by Thomas Bateman on 3 May 1848 from an Anglo-Saxon tumulus (or \"barrow\") at the Benty Grange Farm in the civil parish of Monyash in the English county of Derbyshire.']], ['Jewellery', ['Jewellery (British English) or jewelry (American English) consists of small decorative items worn for personal adornment, such as brooches, rings, necklaces, earrings, pendants and bracelets. Jewellery may be attached to the body or the clothes, and the term is restricted to durable ornaments, excluding flowers for example.', ' For many centuries metal, often combined with gemstones, has been the normal material for jewellery, but other materials such as shells and other plant materials may be used.', ' It is one of the oldest type of archaeological artefact – with 100,000-year-old beads made from \"Nassarius\" shells thought to be the oldest known jewellery.', ' The basic forms of jewellery vary between cultures but are often extremely long-lived; in European cultures the most common forms of jewellery listed above have persisted since ancient times, while other forms such as adornments for the nose or ankle, important in other cultures, are much less common.']], ['Morphology (archaeology)', ['Morphology in archaeology, the study of shapes and forms, and their grouping into period styles remains a crucial tool, despite modern techniques like radiocarbon dating, in the identification and dating not only of works of art but all classes of archaeological artefact, including purely functional ones (ignoring the question of whether purely functional artefacts exist).', ' The term morphology (\"study of shapes\", from the Greek) is more often used for this.', ' Morphological analyses of many individual artefacts are used to construct typologies for different types of artefact, and by the technique of seriation a relative dating based on shape and style for a site or group of sites is achieved where scientific absolute dating techniques cannot be used, in particular where only stone, ceramic or metal artefacts or remains are available, which is often the case.', ' That artefacts such as pottery very often survive only in fragments makes precise knowledge of morphology even more necessary, as it is often necessary to identify and date a piece of pottery from only a few sherds.']], ['Ngườm', ['Ngườm is an archaeological site in Thái Nguyên Province, northern Vietnam.', ' It is a rock shelter in a limestone cliff near the Thần Sa River that was excavated in 1981 by archaeologists from the Vietnam Institute of Archaeology.', ' Flaked stone artefacts have been found in deposits containing shells with radiocarbon ages of 23,000 years ago.', ' The site is important because of its unusually high proportion of retouched flakes in the stone artefact assemblage, relative to other sites in Southeast Asia.']], ['Joe Bell Site', ['The Joe Bell Site (9MG28) is an archaeological site located in Morgan County, Georgia underneath Lake Oconee, but prior to the 1970s, it was located south of the mouth of the Apalachee River on the western bank of the Oconee River.', ' The junction of these two rivers could be seen from the site.', ' This site was first visited by Marshall Williams in 1968 at the suggestion of the site’s landowner, Joe Bell, who had discovered various artifacts while the site was being regularly plowed.', ' Because of Interstate construction, Marshall Williams and Mark Williams discovered this site during surface surveys and excavations of the plowed areas.', ' The site was excavated and analyzed by Mark Williams as part of his PhD dissertation.', ' During the 1969 excavations, four areas within the site were designated for excavation.', ' In Areas 1-3 various five foot square units were excavated.', ' No excavations were done in Area 4 in 1969.', ' Large quantities of small potsherds were discovered during these excavations, and they ranged from the Duvall Phase in Area 1 to Bell Phase in Areas 2-4.', ' As part of the 1969 excavations, a road grading machine took off the topsoil of twelve strips on the site.', ' This uncovered Features 1 and 2, and they were completely excavated.', ' In 1977, the site was revisited by Marshall Williams and Mark Williams.', ' Since various plans threatened this site, major excavations took place from June 15, 1977 until September 16, 1977 by Mark Williams.', ' Most of the work centered on Area 2 or the Bell Phase portion of the site.', ' The Bell Phase portion of this site was probably no more the 1.5 acres .', ' Because of time constraints, only 17 of 55 features were excavated, and no more than a handful of the 1100 posts were excavated.', ' A few trips were made back to the site the following year with the help of volunteers, and approximately 80% of the area stripped by heavy machinery was mapped.', ' Some of the features were trash features that consisted of a circular pit filled with food residues and pottery sherds.', ' Evidence of a large circular structure or rotunda was found at the site.', ' It was the social, political, and religious center for the inhabitants.', ' A large quantity of the features was small, circular, semi-subterranean structures that were probably used as sleeping quarters on cold nights.', ' Another structure found was warm weather structures.', ' One major trash feature was found that had been deposited in a single episode and was burned during or after deposition.', ' Numerous sherds were found in this pit, and a large number of reconstructable vessels were present.', ' Ethnohistoric literature of the Southeast suggests that this feature was formed during a Busk or Green Corn ceremony.', ' The ceremony has been described as the physical cleansing of the town.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.730\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae76e615542997ec2727647', 'answer': '1967', 'question': 'Alternative facts is a phrase use by this U.S. Counselor to the President born in what year?', 'supporting_facts': [['Alternative facts', 0], ['Kellyanne Conway', 0]], 'context': [['Bowling Green massacre', ['The Bowling Green massacre is a fictitious incident alluded to by U.S. Counselor to the President Kellyanne Conway in interviews with \"Cosmopolitan\" and TMZ on January 29, 2017, and in an interview on the MSNBC news program \"Hardball with Chris Matthews\" on February 2, 2017.', ' Conway cited it as justification for a travel and immigration ban from seven Muslim-majority countries enacted by United States President Donald Trump.', ' However, no such massacre occurred.', ' The day after the interview, Conway said she misspoke and had been referring to the 2011 arrest of two Iraqi refugees in Bowling Green, Kentucky on charges including \"attempting to provide material support to terrorists and to al-Qaeda in Iraq\".', ' She stated that she had mentioned the incident because it led President Barack Obama to tighten immigration procedures for Iraqi citizens.']], ['Alternative facts', ['\"Alternative facts\" is a phrase used by U.S. Counselor to the President Kellyanne Conway during a \"Meet the Press\" interview on January 22, 2017, in which she defended White House Press Secretary Sean Spicer\\'s false statement about the attendance numbers of Donald Trump\\'s inauguration as President of the United States.', ' When pressed during the interview with Chuck Todd to explain why Spicer \"utter[ed] a provable falsehood\", Conway stated that Spicer was giving \"alternative facts\".', ' Todd responded, \"Look, alternative facts are not facts.', ' They\\'re falsehoods.\"']], ['Wendy Sherman', [\"Wendy Ruth Sherman (born 1949) is Senior Counselor at Albright Stonebridge Group and also Senior Fellow at Harvard Kennedy School's Belfer Center for Science and International Affairs.\", ' She served as Under Secretary of State for Political Affairs, the fourth-ranking official in the U.S. Department of State, from September 2011 to October 2015.', \" She has formerly worked as a social worker, the director of EMILY's list, the director of Maryland's office of child welfare, and the founding president of the Fannie Mae Foundation.\", ' During the Clinton Administration, she served as Counselor of the United States Department of State and Special Advisor to the President and Secretary of State and North Korea Policy Coordinator.', \" In the latter role, she was instrumental in negotiations related to North Korea's nuclear weapon and ballistic missile programs.\", ' She was also the lead negotiator for the Iran nuclear deal.']], ['Drug Facts for Young People', ['Drug Facts For Young People is an English-language magazine published annually by Regional Maple Leaf Communications Inc.', ' It was first published in 1986 and is aimed at young teens.', ' Drug Facts For Young People focuses on making young people aware of their own values, the influences of their peers and role models, and encourages them to make a positive choice regarding drugs and alcohol.', ' Some schools in US and Canada use the book as an extra curriculum activity.', ' Drug Facts For Young People has been endorsed by the RCMP Foundation since 1999 and was formerly illustrated by Ben Wicks from 1996 - 2000.', ' After his death, RMC created \"The Ben Wicks Award\" in his honor and each year, up-and coming artists from Canada and the US entered the contest for a chance to win the right to illustrate the book and collect the $10,000 prize.']], ['Kellyanne Conway', ['Kellyanne Elizabeth Conway (née Fitzpatrick; born January 20, 1967) is the current Counselor to U.S. president Donald Trump.', ' She is the first woman to ever run a successful presidential campaign.', ' She has been a Republican Party campaign manager, strategist, and pollster, and was formerly president and CEO of The Polling Company Inc./Woman Trend.']], ['Truman State University', ['Truman State University (TSU or Truman) is a public liberal arts and sciences university located in Kirksville, Missouri, United States.', ' It is a member of the Council of Public Liberal Arts Colleges.', ' It had 6,379 enrolled students in the fall of 2015, with 6,039 undergraduate and 340 postgraduate students, pursuing degrees in 48 undergraduate, and eight graduate programs.', ' The university is named after U.S. President Harry Truman, the only president born in Missouri.', \" Until 1996, the school was known as Northeast Missouri State University, but the Board of Trustees voted to change the school's name to better reflect its statewide mission.\", ' In the 2016 U.S. News & World Report College Rankings, Truman placed eighth in the Midwest among regional universities.', ' Truman State is the only public institution in Missouri that is officially designated to pursue highly selective admissions standards.']], ['Alternative facts (law)', ['Alternative facts is a term in law to describe inconsistent sets of facts put forth by the same party in a court given that there is plausible evidence to support both alternatives.', ' The term is also used to describe competing facts for the two sides of the case.']], ['Frederic Adrian Delano', ['Frederic Adrian Delano II (September 10, 1863 – March 28, 1953) was an American railroad president born in Manhattan, New York.', ' He was a member of the Delano family as a son of Warren Delano Jr. and Catherine Robbins Lyman, brother of Sara Ann Delano, and uncle of U.S. President Franklin Delano Roosevelt.', \" His philanthropic work through the Commercial Club of Chicago strongly impacted his nephew's Presidential policies.\", ' Delano was Chairman of the Committee on the Regional Plan for New York and Its Environs, which released the regional plan for New York on May 27, 1929.', ' He was also a prime organizer of the \"Regional Plan for New York and Its Environs,\" published in 1928.', ' He was also a member of the Commercial Club of Chicago which affected the development of Chicago in the 19th and 20th centuries.', ' Delano was the first vice-chairman of the Federal Reserve and the National Resources Planning Board.']], ['James Buchanan', ['James Buchanan Jr. ( ; April 23, 1791June 1, 1868) was the 15th President of the United States (1857–61), serving immediately prior to the American Civil War.', ' He is the only president from Pennsylvania, the only president to remain a lifelong bachelor, and the last president born in the 18th century.', ' A member of the Democratic Party, he was the 17th United States Secretary of State and served in the United States Senate and United States House of Representatives.']], ['Mary N. Cook', ['Mary Nielsen Cook (born June 8, 1951) was a counselor in the general presidency of the Young Women organization of The Church of Jesus Christ of Latter-day Saints (LDS Church) from 2007 to 2013.', ' She served as second counselor to Susan W. Tanner, with Elaine S. Dalton as first counselor, from March 2007 until April 2008.', ' In April 2008, Dalton succeeded Tanner as Young Women General President and selected Cook as her first counselor.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.730\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a75404a55429916b01642be', 'answer': 'Syracuse University', 'question': 'At what university can the building that served as the fictional household that includes Gomez and Morticia be found?', 'supporting_facts': [['Hall of Languages, Syracuse University', 0], ['Hall of Languages, Syracuse University', 3], ['The Addams Family', 1]], 'context': [['J. L. Robinson General Store', ['The J. L. Robinson General Store is a historic general store located on Hagaman Road in Hagaman, Macoupin County, Illinois.', \" Railroad worker Charles Crossland Robinson opened the store in 1881; his son James Leo Robinson took over the store upon his father's death, giving it its current name.\", ' The store was the main seller of household and farming supplies in Western Mound Township and the Hagaman and Chesterfield areas.', \" In addition, the building served as the township's post office, a railway express office for Macoupin County's two railroads, and the local Democratic Party headquarters.\", ' The store was also the first building in the area to receive electricity and telephone service.', ' It is now the only surviving building in the area built before 1900.']], ['Bobby Mehta', ['Siddharth N. \"Bobby\" Mehta was former CEO and vice chairman of HSBC North America.', ' Mehta served as an Advisor of TransUnion since December 31, 2012.', ' Mehta serves as consultant of TransUnion.', ' He served the chief executive officer and president of TransUnion from August 2007 to December 31, 2012, and Transunion Financing Corp. until December 31, 2012.', ' From May 2007 to July 2007, he served as a consultant to the board of directors at TransUnion.', ' He served as the chief executive officer and president of TransUnion until December 31, 2012.', ' He served as the chief executive officer of TransUnion LLC.', ' He served as chairman of the board and chief executive officer of HSBC Finance Corporation from April 2005 to February 2007.', ' He served as chief executive officer and president of TransUnion LLC from 2007 to 2012.', ' From 1998 to 2007, he held a variety of positions with HSBC Finance Corporation and HSBC North America Holdings, Inc.', ' Mehta served as chief executive officer of HSBC North America until February 2007.', ' Mehta served as consultant of TransUnion since May 2007 until July 2007.', ' Mehta served as group managing director of HSBC Holdings PLC of HSBC Finance Corp. since April 30, 2005, and its unit chief executive officer since March 2005.', ' He served as the chief executive of HS BC North America Holdings Inc., of HSBC Finance Corp., from March 2005 to February 15, 2007.', ' He served as an executive chairman of HSBC Financial Corporation Limited since April 2005 and served as its chief executive officer from April 2005 to February 15, 2007.', ' He served as the chief executive officer of HSBC Bank USA, N.A. until February 2007.', ' He served as the chief executive officer of HSBC North America Holdings Inc. since March 2005.', ' He served as chairman and chief executive officer of HSBC Financial Corp., Ltd.', \" He oversaw HSBC's global credit card services, its North American consumer lending and mortgage services businesses and its first mortgage operation.\", ' He was also responsible for corporate marketing, strategic planning and corporate development for HSBC North America Holdings Inc. and had responsibility for the strategic management of credit cards throughout the HSBC Group.', ' Mehta served as group executive of Credit Card Services, Auto Finance and Canada of Household International Inc., since July 2002.', ' He worked at MasterCard’s U.S. region board since March 2000.', ' Mehta joined Household International Inc., in 1998.', ' He served as senior vice president of The Boston Consulting Group in Los Angeles and co-leader of Boston Consulting Group Financial Services Practice in the United States.', ' Mehta served as a director of Global Board of MasterCard Incorporated since March 17, 2005.', ' He served as unit chairman of HSBC Holdings PLC and served as its board member since March 2005.', ' He served as vice chairman and director of HSBC Financial Corporation Limited., (Formerly Household International Inc.).', ' He has been a director of Avant Credit Corporation since December 18, 2014.', ' He has been an independent director of The Allstate Corporation since February 19, 2014.', ' He serves as a member of the advisory board at Core2 Group, Inc.', ' He has been non-executive independent director at Piramal Enterprises Ltd since April 1, 2013.', ' He serves on the boards of Datacard, Chicago Public Education Fund, University of Chicago Laboratory Schools, The Economic Club of Chicago, The Field Museum and Myelin Repair Foundation.', ' He serves as a director of TransUnion Corp. and TransUnion LLC.', ' He served as a director of MasterCard International Inc. (also known as MasterCard Worldwide) (formerly, MasterCard Inc.), since March 17, 2005.', ' He served as a director of HSBC Financial Corp.', ' Ltd.', ' He has been a director of TransUnion since April 2012.', ' Mehta serves on the board of international advisors for the Monterey, California, Institute of International Studies and is a member of the Financial Services Roundtable.', ' He also serves on the board of advisors for the Myelin Repair Foundation.', ' Mehta holds a Bachelor of Arts in economics from the London School of Economics and Masters of Business Administration from the University of Chicago.', ' He stepped down as head of the North American unit after the lender raised its forecast for bad loans in the U.S.', ' He is of Indian descent.']], ['Concealed shoes', ['Concealed shoes hidden in the fabric of a building have been discovered in many European countries, as well as in other parts of the world, since at least the early modern period.', ' Independent researcher Brian Hoggard has observed that the locations in which these shoes are typically found\\xa0– in chimneys, under floors, above ceilings, around doors and windows, in the roof\\xa0– suggest that some may have been concealed as magical charms to protect the occupants of the building against evil influences such as demons, ghosts and witches.', ' Others may have been intended to bestow fertility on a female member of the household, or been an offering to a household deity.']], ['Le Ménagier de Paris', ['Le Ménagier de Paris (often abbreviated as Le Ménagier, and meaning \"\"The Parisian Household Book\"\") is a French medieval guidebook from 1393 on a woman\\'s proper behaviour in marriage and running a household.', ' It includes sexual advice, recipes, and gardening tips.', ' Written in the (fictional) voice of an elderly husband addressing his younger wife, the text offers a rare insight into late medieval ideas of gender, household, and marriage.', \" Important for its language and for its combination of prose and poetry, the book's central theme is wifely obedience.\"]], ['Hall of Languages, Syracuse University', ['The Hall of Languages is a Syracuse University building designed by Horatio Nelson White in the Second Empire architectural style, and built in 1871–73.', ' It is made of Onondaga limestone and features three large towers or cupolas.', ' It was the first building constructed on the Syracuse University campus and the building originally housed the entire university.', ' The building served as creative inspiration for the Addams Family home.']], ['Kamp Store', ['The Kamp Store is a historic general store building located at the northeast corner of Oak and Broadway in Kampsville, Illinois, United States.', ' Joseph Kamp, the son of the founder of Kampsville, opened the store in 1902.', ' The two-story wood frame building features a false front with decorative metalwork.', ' The store provided Kampsville residents with a wide variety of goods, ranging from small household items to automobiles and heavy farming equipment.', ' St. Louis-based suppliers shipped the store its goods via Mississippi River barges.', ' Kamp operated the store until his death in 1952; the store served as a grocery store until the 1970s and later became a carpet store.', \" The Center for American Archeology purchased the building in 1991 and now uses it as its Visitor's Center and Museum.\"]], ['The Addams Family', ['The Addams Family is a fictional household created by American cartoonist Charles Addams.', \" The Addams Family characters have traditionally included Gomez and Morticia Addams, their children Wednesday and Pugsley, close family members Uncle Fester and Grandmama, their butler Lurch, the disembodied hand Thing, and Gomez's Cousin Itt.\"]], ['The Powerhouse (San Luis Obispo, California)', ['The Powerhouse is a historic building located on the campus of California Polytechnic State University in San Luis Obispo, California.', ' Built from 1908 to 1910, the building was designed by William H. Weeks in the Mission Revival style.', ' The Powerhouse was the last of the original buildings at Cal Poly to be constructed; however, it is now the only remaining original building on its campus.', ' The building originally served as a power plant run by students and two full-time supervisors; it also held Mechanics and Electrical Engineering classes.', ' The Powerhouse stopped generating power in the 1940s and was replaced entirely and abandoned in 1955.', \" In 1967, the building found a new use when the school's College of Architecture and Environmental Design decided to hold classes there.\", \" The college continued to hold classes in the building even after the construction of a new architecture building, and only stopped in 1990 when the school's administration ordered the building to be abandoned.\"]], ['Sanheyuan', ['Sanheyuan (Chinese: 三合院; pinyin: \"sānhéyuàn\"; Wade–Giles: \"san-ho-yüan\") is a historical type of residence that was commonly found throughout China.', ' Sanheyuan have structures on three sides of a courtyard, forming an inverted U-shape, resembling the Chinese character 凹 (pinyin: \"āo\").', ' There is normally a wall linking the two forward-thrusting side wings, called 廂房 (pinyin: \"xiāngfáng\"), similar to the wings of a siheyuan.', ' Sanheyuan may be square or rectangular shaped and can be single or multiple-story structures.', ' Typically there are three structural divisions within the horizontal building in the U, which is called a \"three-jian\" building (Chinese: 三間屋; pinyin: \"sānjiānwū\").', ' The purpose of this main building varies by region, but typically consists of a central room serving ceremonial needs flanked on either side by a bedroom.', ' The two wings making up the arms of the U may be long or short, according to need, and provide room for kitchens, toilets, storage, and additional bedrooms.', ' In Taiwan, the wings of the sanheyuan are called \"protecting dragons\" (Chinese: 護龍; pinyin: \"hùlóng\").', ' Additional \"hulong\" would typically be added in pairs, placed parallel to the first set and then duplicated as the household grew.', ' The inner pair of \"hulong\" were traditionally called \"inner protectors\" (Chinese: 內護; pinyin: \"nèihù\"), the second pair \"outer protectors\" (Chinese: 外護; pinyin: \"wàihù\"), and so on.']], ['Waste House', ['Waste House is a building on the University of Brighton campus in the centre of Brighton on the south coast of England.', ' It was built between 2012 and 2014 as a project involving hundreds of students and apprentices and was designed by Duncan Baker-Brown, an architect who also lectures at the university.', ' The materials consist of a wide range of construction industry and household waste—from toothbrushes and old jeans to VHS cassettes and bicycle inner tubes—and it is the first public building in Europe to be built primarily of such products.', ' \"From a distance [resembling] an ordinary contemporary town house\", Waste House is designed to be low-energy and sustainable, and will be in continuous use as a test-bed for the university\\'s design, architecture and engineering students.', \" The building has won several awards and was shortlisted for the Royal Institute of British Architects' Stephen Lawrence Prize in September 2015.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.732\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7a77425542995eb53be83e', 'answer': 'Brady Seals', 'question': 'Who has released more solo albums, Nick Carter or Brady Seals?', 'supporting_facts': [['Nick Carter (musician)', 0], ['Nick Carter (musician)', 2], ['Brady Seals', 0], ['Brady Seals', 5]], 'context': [['The Truth (Brady Seals album)', ['The Truth is the debut studio album of American country music singer Brady Seals, and his first solo album after leaving the country band Little Texas.', ' It was released on February 25, 1997 on Reprise Records.', ' The album includes the singles \"Another You, Another Me\", \"Still Standing Tall\", and \"Natural Born Lovers\".', ' Of these, only \"Another You, Another Me\" reached Top 40 on Hot Country Songs.', \" Seals co-wrote all but two of the album's songs, and co-produced the album with Rodney Crowell.\"]], ['Nick Carter, Master Detective', [\"Nick Carter, Master Detective was a Mutual radio crime drama based on tales of the fictional private detective Nick Carter from Street & Smith's dime novels and pulp magazines.\", \" Nick Carter first came to radio as The Return of Nick Carter, a reference to the character's pulp origins, but the title was soon changed to Nick Carter, Master Detective.\", ' A veteran radio dramatist, Ferrin Fraser, wrote many of the scripts.']], ['Another You, Another Me', ['\"Another You, Another Me\" is a debut solo song recorded by American country music artist Brady Seals.', ' It was released in September 1996 as the first single from the album \"The Truth\".', ' The song reached #32 on the \"Billboard\" Hot Country Singles & Tracks chart.', \" The song was written by Seals' uncle, Troy Seals, along with Will Jennings.\"]], ['Brady Seals', ['Brady Seals (born March 29, 1969) is an American country music artist.', ' He is the cousin of Jim Seals (of Seals & Crofts) and Dan Seals and Johnny Duncan, the nephew of Troy Seals, and the husband of former BNA Records recording artist Lisa Stewart.', ' Seals made his debut in 1988 as co-lead vocalist and keyboardist in the sextet Little Texas, with whom he recorded until his departure in late 1994.', ' Between then and 2002, he recorded as a solo singer, releasing three studio albums and charting in the Top 40 on the country charts with \"Another You, Another Me\".', ' In 2002, Seals formed a quartet called Hot Apple Pie, in which he has recorded one studio album and charted three singles.', ' A fourth solo album, \"Play Time\", was released in 2009 via Star City.']], ['Nick Carter va tout casser', ['Nick Carter va tout casser is a French action film starring Eddie Constantine as Nick Carter.', ' An English version was dubbed by Eddie Constantine dubbing himself.', ' Constantine repeated his role in \"Nick Carter et le trèfle rouge\" (1965).', ' The film was titled License to Kill in the USA.']], ['Nick Carter (musician)', ['Nickolas Gene \"Nick\" Carter (born January 28, 1980) is an American singer, songwriter, actor, and dancer.', ' He is best known as a member of the pop group the Backstreet Boys.', ' As of 2015, Carter has released three solo albums, \"Now or Never\", \"I\\'m Taking Off\" and \"All American\" during breaks between Backstreet Boys schedules, and a collaboration with Jordan Knight titled \"Nick & Knight\".', ' He has made occasional television appearances and starred in his own reality shows, \"House of Carters\" and \"I (Heart) Nick Carter\".', ' He gained fame in the mid 1990s and early 2000s as a teen idol.', ' He is also the older brother of singer Aaron Carter and the late Leslie Carter.']], ['My Love (Little Texas song)', ['\"My Love\" is a song recorded by the American country music band Little Texas.', \" It was co-written by the band's keyboardist Brady Seals (who also sang lead vocals on it) and lead guitarist Porter Howell along with Tommy Barnes.\", ' It was released in January 1994 as the third single from the album, \"Big Time\".', ' The song reached the top of the \"Billboard\" country singles charts, becoming the band\\'s only Number One country hit.', \" The song features lead vocals from Brady Seals, then the band's keyboardist.\"]], ['Brady Seals (album)', ['Brady Seals is the self-titled second album by American country music singer Brady Seals.', ' It is his second release independently of the band Little Texas, of which he was a member until 1995.', ' The album includes the singlse \"I Fell\", \"Whole Lotta Hurt\" and \"The Best Is Yet to Come\".', ' All three singles charted on the \"Billboard\" country charts, although they all missed Top 40.']], ['Nick Carter and Red Club', ['Nick Carter and Red Club (French: \"Nick Carter et le trèfle rouge\" ) is a 1965 French action film directed by .', ' The film features the successful literary character Nick Carter and is based on a novel by Claude Rank.', ' The film is a sequel to \"Nick Carter va tout casser\" (1964).']], ['Thompson Street (album)', ['Thompson Street is the third solo album by American country music singer Brady Seals.', ' It was released in February 2003 via Image Entertainment.', ' No singles were released from it, and after its release, Seals founded the band Hot Apple Pie.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.733\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8b3de95542997f31a41cc3', 'answer': 'Tian Tan Buddha', 'question': 'In what year was the most famous statute at Po Lin Monastery built?', 'supporting_facts': [['Buddhism in Hong Kong', 1], ['Tian Tan Buddha', 0]], 'context': [['Bagaya Monastery', ['The Bagaya Monastery (Burmese: ဘားဂရာ ကျောင်း ), located in Inwa, Mandalay Region, Burma (Myanmar) is a Buddhist monastery built on the southwest of Inwa Palace.', ' This magnificent monastery is also known as Maha Waiyan Bontha Bagaya Monastery.', \" During King Hsinbyushin's reign (1763–1776), Maha Thiri Zeya Thinkhaya, town officer of Magwe built the monastery in the Bagaya monastic establishment and dedicated to Shin Dhammabhinanda.\", ' It is one of the famous tourists attractions in Burma.']], ['Buddhism in Hong Kong', ['Buddhism is a major religion in Hong Kong and has been greatly influential in the traditional culture of its populace.', \" Among the most prominent Buddhist temples in the city there are the Chi Lin Nunnery in Diamond Hill, built in the Tang Dynasty's architectural style; the Po Lin Monastery on Lantau Island, famous for the outdoor bronze statue, Tian Tan Buddha, which attracts a large number of visitors during the weekends and holidays.\"]], ['Po Lin Monastery', ['Po Lin Monastery is a Buddhist monastery, located on Ngong Ping Plateau, on Lantau Island, Hong Kong.']], ['Ngong Ping 360', ['The Ngong Ping 360 is a gondola lift on Lantau Island in Hong Kong.', ' Intended to improve tourism to the area, the aerial lift was previously known as Tung Chung Cable Car Project before acquiring the Ngong Ping 360 brand in April 2005.', \" It consists of the Ngong Ping Cable Car, formerly known as the Ngong Ping 360 Skyrail, and the Ngong Ping Village, a retail and entertainment centre adjacent to the cable car's upper station.\", ' Ngong Ping 360 connects Tung Chung, on the north coast of Lantau and itself linked to central Hong Kong by the Tung Chung Line, with the Ngong Ping area in the hills above.', ' This is home to the Po Lin Monastery and the Tian Tan Buddha, both already significant tourist attractions in their own right.', \" Before Ngong Ping 360's opening, the only access was via a mountain road and bus service.\"]], ['Great Lavra', ['The Monastery of Great Lavra (Greek: Μονή Μεγίστης Λαύρας ) is the first monastery built on Mount Athos.', ' It is located on the southeastern foot of the Mount at an elevation of 160 metres.', ' The founding of the monastery in AD 963 by Athanasius the Athonite marks the beginning of the organized monastic life at Mount Athos.', ' At the location of the monastery, there was one of the ancient cities of the Athos peninsula, perhaps Akrothooi, from which the sarcophagi of the monastery that are in the oil storage house come.', ' The history of the monastery is the most complete compared to the history of the other monasteries, because its historical archives were preserved almost intact.', ' It is possible that the study of these archives may contribute to the completion of the knowledge of the history of other monasteries, whose archives were partially or completely lost.']], ['Tian Tan Buddha', ['Tian Tan Buddha, also known as the Big Buddha, is a large bronze statue of Buddha Shakyamuni, completed in 1993, and located at Ngong Ping, Lantau Island, in Hong Kong.', ' The statue is sited near Po Lin Monastery and symbolises the harmonious relationship between man and nature, people and faith.', ' It is a major centre of Buddhism in Hong Kong, and is also a popular tourist attraction.']], ['Enchey Monastery', ['The Enchey Monastery was established in 1909 above Gangtok, the capital city of Sikkim in the Northeastern Indian state.', ' It belongs to the Nyingma order of Vajrayana Buddhism.', ' The monastery built around the then small hamlet of Gangtok became a religious centre.', ' The location was blessed by Lama Drupthob Karpo, a renowned exponent of tantric (adept) art in Buddhism with flying powers; initially a small Gompa was established by him after he flew from Maenam Hill in South Sikkim to this site.', ' The literal meaning of Enchey Monastery is the \"Solitary Monastery\".', ' Its sacredness is attributed to the belief that Khangchendzonga and Yabdean – the protecting deities – reside in this monastery.', ' As, according to a legend, Guru Padmasambhava had subdued the spirits of the Khangchendzonga, Yabdean and Mahākāla here.', ' In view of this legend, the religious significance of Enchey Monastery is deeply ingrained in every household in Gangtok.', ' It is also believed that these powerful deities always fulfil the wishes of the devotees.']], ['Lenggu Monastery', [\"Lenggu Monastery, also transliterated as Rengo Monastery, is a Tibetan Buddhist monastery at the foot of Ge'nyen Mountain in Sichuan, China.\", ' The monastery was formerly known as Kambo Dansar and was the first monastery built by the first Kamapa Duisung Chenpa in 1164.', ' The 7th Kamapa Quzha Qiangcuo enlarged it.', ' At its peak, over 2000 monks studied at the monastery.', ' Today, Lengu is administered as part of Zhamla Township in Litang County, Garze Prefecture, Sichuan.']], ['Nei Lak Shan', ['Nei Lak Shan (Chinese: 彌勒山) is the sixth highest hill in Hong Kong.', ' With a height of 751 m on Lantau Island, it is situated immediately north of Ngong Ping where the Buddhist Po Lin Monastery is located.', ' \"Nei Lak\", or correctly \"Mei Lak\" is a Cantonese language translation of Maitreya, the future Buddha, in Buddhism.']], ['Ngong Ping', ['Ngong Ping () is a highland in the western part of Lantau Island, Hong Kong.', ' It hosts Po Lin Monastery and Tian Tan Buddha amidst the hills which is about 34 m tall.', ' There are several hills nearby which are also an attraction to tourists.', ' It is now the terminus of the cable car ride Ngong Ping 360 which travels to Tung Chung.', \" New facilities and tourist attractions have opened including the Ngong Ping Village, Walking with the Buddha, the Monkey's Tale Theatre and Ngong Ping Tea House.\", ' A youth hostel is located near the monastery.', ' The second highest peak of Hong Kong, Lantau Peak, is at its southeast.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.733\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a728dc65542992359bc30ef', 'answer': 'Nihat Kahveci', 'question': 'What fellow Turkish footballer did Darko Kovačević form an offensive partnership with during his tenure at Real Sociedad?', 'supporting_facts': [['Darko Kovačević', 3], ['Nihat Kahveci', 0]], 'context': [['Real Sociedad Femenino', [\"Real Sociedad Femenino is the women's football section of Real Sociedad de Fútbol.\", ' Founded in 2004']], ['Lippo Hertzka', ['Lippo Hertzka (19 November 1904 – 14 March 1951) was a Hungarian football player and manager.', ' He played for Essener Turnerbund, MTK Budapest and Real Sociedad.', ' After retiring, he coached seven teams, including Real Sociedad and Real Madrid, a team which he coached for 2 years (1930–1932) and led to an undefeated La Liga championship during the 1931-32 season, which meant the first La Liga title for the \"white\" squad.', ' He also won two league titles in Portugal for Benfica.']], ['Adnan Januzaj', ['Adnan Januzaj (] ; born 5 February 1995) is a Belgian professional footballer who plays as a winger for Spanish club Real Sociedad.', ' Born and raised in Brussels, he began his career with Anderlecht before joining Manchester United in 2011 at the age of 16.', \" Januzaj broke into the Manchester United first-team under manager David Moyes during the 2013–14 season, but struggled for opportunities under Moyes' successors Louis van Gaal and José Mourinho, and had loan spells at Borussia Dortmund and Sunderland before joining Real Sociedad in July 2017.\", ' Januzaj made his full international debut in 2014 and later that year played for Belgium at the World Cup.']], ['Real Sociedad', ['Real Sociedad de Fútbol, S.A.D., more commonly referred to as Real Sociedad (] ; \"Royal Society\") or La Real, is a Spanish football club based in the city of San Sebastián, Basque Country, founded on 7 September 1909.', ' It plays its home matches at the 32,000-capacity Anoeta Stadium.', ' Real Sociedad won the Liga title in 1980–81 and 1981–82, and last finished runners-up in 2002–03.', ' The club has also won the Copa del Rey twice, in 1909 and 1987.', ' It contests the Basque derby against rivals Athletic Bilbao.', ' Real Sociedad were founder members of La Liga in 1928, and its longest spell in the top flight was for 40 seasons, from 1967 to 2007.']], ['José Antonio Santamaría Mikel Vaqueriza', ['José Antonio Santamaría Mikel Vaqueriza (born 16 March 1946 in San Sebastián, Guipúzcoa, Spain; died 19 January 1993) was a Spanish footballer.', ' During his career as a professional footballer, Santamaría played for SD Eibar (1963 to 1964), Real Sociedad (1964 to 1967), Real Sociedad (1967 to 1971), Hercules CF (1971 to 1974) finishing his career at CE Sabadell FC (1974 to 1975).', ' After retiring from football he had a career as an entrepreneur in the hospitality sector.', ' He was killed in an attack committed by ETA in 1993.']], ['C.D. Real Sociedad', ['Club Deportivo Real Sociedad, commonly known as Real Sociedad (] ), is a Honduran football club based on Tocoa, Colón, Honduras.']], ['Sport Mundi Tournament', [\"The Sport Mundi Tournament is a Spanish pre-season women's football invitational charitable tournament held every August in Irun since 2005.\", ' It is contested by four teams, usually including regional powerhouses Athletic Bilbao and Real Sociedad.', ' Athletic is the most successful team in the competition with four titles, followed by Levante UD and Real Sociedad with two each.']], ['2013–14 Real Sociedad season', [\"The 2013–14 season was Real Sociedad's 67th season in La Liga.\", ' Real Sociedad finished 7th in the league and reached the semifinals of the Copa del Rey.', ' The Basque failed to make it out of the group stages of the UEFA Champions League.']], ['Darko Kovačević', ['Darko Kovačević (] ; Serbian Cyrillic: ; born 18 November 1973) is a Serbian former footballer who played as a forward.', ' He began his career in Serbia with Proleter Zrenjanin and subsequently played for Red Star Belgrade, with whom he won a Yugoslav League title and two Yugoslav Cups.', ' His prolific performances earned him a move to Premier League side Sheffield Wednesday, although his time in England was less successful.', ' He is mainly known for his spells at Real Sociedad where his offensive partnership with Nihat Kahveci was one of the best in Spain.', ' Kovačević also had positive spells with Italian club Juventus and Greek side Olympiacos.', ' At international level, he represented Yugoslavia at the 1998 FIFA World Cup and at UEFA Euro 2000.']], ['Nihat Kahveci', ['Nihat Kahveci (born 23 November 1979) is a Turkish former footballer who played as a forward.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.734\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae1338d55429901ffe4ade1', 'answer': 'the green fairy', 'question': \"The alcoholic drink that shares a name with Quintessentially Unreal's 1996 album is commonly reffered to in historical literature as what?\", 'supporting_facts': [['Quintessentially Unreal', 0], ['Absinthe', 2]], 'context': [['List of alcoholic drinks', ['This is a list of alcoholic drinks.', ' An alcoholic drink is a drink that contains ethanol, commonly known as alcohol.', ' Alcoholic drinks are divided into three general classes: beers, wines, and distilled beverages.', ' They are legally consumed in most countries, and over one hundred countries have laws regulating their production, sale, and consumption.', ' In particular, such laws specify the minimum age at which a person may legally buy or drink them.', ' This minimum age varies between 16 and 25 years, depending upon the country and the type of drink.', ' Most nations set it at 18 years of age.']], ['Arak (drink)', ['Arak or araq (Arabic: عرق\\u200e \\u200e ) is a Levantine alcoholic spirit (~40–63% Alc.', ' Vol.', '/~80–126 proof, commonly 50% Alc.', ' Vol.', '/100 proof) in the anis drinks family.', ' It is a clear, colorless, unsweetened anise-flavored distilled alcoholic drink.', ' The Persian (Iranian) version of Arak (commonly called Arak Saggi) does not contain anise, as it is usually produced from \\u200craisins, dates or saccharum plant.', ' Arak is the traditional alcoholic beverage in the Arab world, especially in the Levant/Mashriq and also in the Maghreb, as well in Iran and Turkey.']], ['Marskin ryyppy', ['Marskin ryyppy (lit.', ' \"The Marshal\\'s drink/shot\"; Swedish: \"Marskens snaps\" ) is a strong alcoholic drink of Finnish origin, served as a schnapps.', ' The drink is named after Carl Gustaf Emil Mannerheim, the Marshal of Finland.', ' According to all sources it is important that the glass where the drink is served is poured as full as possible (to the point where surface tension keeps some of the drink in the glass); the glass is then to be emptied without spilling.', ' This practice is said to originate in the Chevalier Guard where Mannerheim once served; every man was entitled one shot of Vodka per day, and through this practice everyone was assured the equal maximum amount.', ' Marskin ryyppy must also be served ice-cold.']], ['Zurracapote', ['Zurracapote (sometimes abbreviated as zurra) is a popular Spanish alcoholic mixed drink, similar to sangría.', ' It consists of red wine mixed with fruit such as peaches and lemons, sugar, and cinnamon.', ' The concoction is then traditionally left to steep for several days, though some recipes call for the addition of other alcoholic beverages, juices, and fruit extracts.', ' The result is a mild-to-medium alcoholic drink, similar to sangría.']], ['Cundill Prize', ['The Cundill Prize in Historical Literature (or simply Cundill Prize) was founded in 2008 by Peter Cundill to recognize and promote literary and academic achievement in history.', ' The prize is presented annually to an author who has published a non-fiction book in the prior year that is likely to have profound literary, social, and academic impact in the area of history.', ' At a value of $75,000 U.S., the Grand Prize is claimed to be the richest non-fiction historical literature prize in the world.', ' In addition, two \"Recognition of Excellence\" prizes of $10,000 U.S. each are awarded.', ' The winners of the Prizes are selected by an independent jury of at least five internationally qualified individuals selected by McGill University.', \" The Cundill Prize in History at McGill is administered by McGill University's Dean of Arts, with the help of the McGill Institute for the Study of Canada (MISC).\"]], ['Quintessentially Unreal', ['Quintessentially Unreal is the debut album by American Neo-Cabaret artist Jill Tracy, released in 1996.', ' It was nominated for California Music Awards in 1997 and 1998.', ' Selections from the album were used on an NBC Hard Copy segment on Absinthe.']], ['Caffeinated alcoholic drink', ['A caffeinated alcoholic drink, or caffeinated alcoholic beverage, is a drink that contains both alcohol (also known formally as ethanol) and caffeine.', ' They often include the ingredients of energy drinks as well.', ' In 2010 and 2011, this type of beverage faced criticism for posing health risks to their drinkers.', ' In some places there is a ban on caffeinated alcoholic beverages.']], ['Rakı', ['Rakı is an unsweetened, occasionally (depending on area of production) anise-flavored, alcoholic drink that is popular in Albania, Turkey, Greece (where it is distinctly different and comes as an unflavoured distillate, unlike its Turkish counterpart), Iran, Turkic countries, and in the Balkan countries as an apéritif.', ' It is often served with seafood or meze.', ' It is similar to several other alcoholic beverages available around the Mediterranean and the Middle East, e.g. pastis, ouzo, sambuca, arak, Aragh Sagi and aguardiente.', ' In Turkey and Greece, it is considered a national drink.']], ['Tempore', ['Tempore (abbreviated to temp.)', ' in historical literature denotes a period during which a person whose exact lifespan is unknown was known to have been alive or active, or some other date which is not exactly known, usually given as the reign of a monarch.', ' The word is Latin, being the ablative singular of the noun \"tempus, temporis\", \"time\", thus meaning \"in the time (of)\".', ' It should be followed by a name in the genitive case.', ' The theoretical full form might be \"vixit tempore Regis Henrici Primi\" (\"he/she lived in the time of King Henry the First\") (i.e. 1100-1135).', ' The best known occurrence is in the Domesday Book of 1086, where the phrase \"Tempore Regis Eduardi\" (nominative case \"Rex Eduardus\"), meaning \"in the time of King Edward (the Confessor)\" appears in the entry for almost every manor, abbreviated as TRE.', ' It thus signifies the date range 1042–1066.', ' It is useful in historical literature because the names of many historical persons appear in surviving documents only in royal charters, possibly as witnesses, which can be dated to the reign of the originating monarch.']], ['Absinthe', ['Absinthe ( or ; French: ] ) is historically described as a distilled, highly alcoholic (45–74% ABV / 90–148 U.S. proof) beverage.', ' It is an anise-flavoured spirit derived from botanicals, including the flowers and leaves of \"Artemisia absinthium\" (\"grand wormwood\"), together with green anise, sweet fennel, and other medicinal and culinary herbs. Absinthe traditionally has a natural green colour but may also be colourless.', ' It is commonly referred to in historical literature as \"\"la fée verte \"\" (the green fairy).', ' Although it is sometimes mistakenly referred to as a liqueur, absinthe is not traditionally bottled with added sugar; it is therefore classified as a spirit.', ' Absinthe is traditionally bottled at a high level of alcohol by volume, but it is normally diluted with water prior to being consumed.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.735\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae56fb85542993aec5ec1a9', 'answer': 'Delphine Software International', 'question': \"What software company developed Fade to black and it's 1992 Sequel, Flashback? \", 'supporting_facts': [['Fade to Black (video game)', 0], ['Flashback (1992 video game)', 0]], 'context': [['Helix Software Company', ['Helix Software Company was a New York City based software company founded in October 1986.', ' The company developed software tools and utilities for DOS and Windows.', ' In 1993, Helix licensed some of its memory management technology to Microsoft for use in MS-DOS 6.0.', \" Microsoft subsequently released Helix's memory management technology as part of the MEMMAKER and EMM386 DOS commands.\"]], ['AVG Technologies', ['AVG Technologies is a security software company headquartered in Amsterdam, Netherlands, that was founded in 1991 by Jan Gritzbach and Tomáš Hofer.', ' Since 2016 the company has been a subsidiary of Avast Software.', ' The company developed antivirus software and internet security services such as AVG AntiVirus and has corporate offices in Europe, Israel, Brazil, Canada and the United States.']], ['Objectory AB', ['Objectory Systems was a software company based in Sweden that was instrumental in the development of Object-oriented program design.', ' Founded in 1987 by Ivar Jacobson, the company developed Objectory, an object-oriented development method which was an extension of what is known as the \"Ericsson Approach\", a modeling language developed at Ericsson.', ' This language featured state charts with activity diagrams, as well as sequence diagrams.']], ['WebMethods', ['webMethods was an enterprise software company, acquired by Software AG, focused on application integration, business process integration and B2B partner integration.', ' Founded in 1996, the company sold systems for organizations to use web services to connect software applications over the Internet.', ' In 2000, the company went public on the NASDAQ in the most successful software IPO to date, based on investor interest and first day share price appreciation.', ' In 2002, the company was named by Deloitte as the fastest-growing software company in North America over the period 1998 to 2002.', ' In 2007 webMethods was acquired by Software AG for $546 million and was made a subsidiary of that company.', ' In 2010 the webMethods division of Software AG recorded over $668 million in revenues.', ' Software AG retained the webMethods name, and uses it as a brand to identify a software suite encompassing process improvement, SOA enablement, IT modernization and business and partner integration.']], ['Flashback (1992 video game)', ['Flashback, released as Flashback: The Quest for Identity in the United States, is a 1992 science fiction cinematic platform game developed by Delphine Software of France and published by U.S. Gold in United States and Europe, and Sunsoft in Japan.']], ['Windmill Software', ['Windmill Software is a Canadian software company.', ' Windmill Software today publishes property management software and management information system software, but the company is more notable for its past role as a developer, marketer, publisher, and distributor of computer and video games.', ' The company developed several games for the IBM PC in the early 1980s.']], ['Fade to Black (video game)', ['Fade to Black is an action-adventure game developed by Delphine Software International and published by Electronic Arts.', ' It is the sequel to the 1992 video game \"Flashback\".', ' The game was released for MS-DOS and PlayStation.', ' Planned Nintendo 64 and Sega Saturn versions were cancelled.']], ['NuMega', ['NuMega Technologies (or NuMega) was a software company founded in 1987 by Frank Grossman and Jim Moskun in Nashua, New Hampshire, USA.', ' The company developed Kernel mode debugger, now SoftICE, for DOS and the Windows NT family.']], ['Sorenson Media', ['Sorenson Media is an American software company specializing in video encoding technology.', ' Established in December 1995 as Sorenson Vision, the company developed technology which was licensed and ultimately acquired from Utah State University.', ' The company first announced its codec (compression and decompression tool) at a developer’s preview at MacWorld Expo in January 1997.']], ['Bare Bones Software', ['Bare Bones Software is a private North Chelmsford, Massachusetts, United States software company developing software tools for the Apple Macintosh platform.', ' The company developed the BBEdit text editor, marketed under the registered trademark \"\"It doesn\\'t suck,\"\" and has been mentioned as a \"top-tier Mac developer\" by Mac OS X journalist John Siracusa.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.736\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a838eba5542992ef85e22fe', 'answer': 'Volcanic Sprint', 'question': 'Which documentary film was released first Tar Creek or Volcanic Sprint?', 'supporting_facts': [['Tar Creek (film)', 0], ['Volcanic Sprint', 0]], 'context': [['Volcanic Sprint', ['Volcanic Sprint is a 2007 documentary film produced by Steve Dorst and Dan Evans about a grueling mountain race in Africa and the mostly local competitors who compete against tough odds to overcome dire poverty.']], ['Single Video Theory', ['Single Video Theory is a music documentary directed by Mark Pellington that follows the making of \"Yield\", the fifth album by the American alternative rock band Pearl Jam.', ' It was released first on VHS on August 4, 1998, and then on DVD on November 24, 1998.']], [\"If God Is Willing and da Creek Don't Rise (film)\", ['If God Is Willing and da Creek Don\\'t Rise is a 2010 documentary film directed by Spike Lee, as a follow-up to his 2006 HBO documentary film, \"\".', ' The film looks into the proceeding years since Hurricane Katrina struck the New Orleans and Gulf Coast region, and also focuses on the 2010 BP oil spill in the Gulf of Mexico and its effect on the men and women who work along the shores of the gulf.', ' Many of the participants in \"Levees\" were also featured in this documentary.']], ['Game (2016 film)', ['Game in Kannada, Oru Melliya Kodu (English: A thin line) in Tamil, is a 2016 Indian bilingual language crime thriller film directed by A. M. R. Ramesh.', ' This movie is an unofficial remake of the 2012 Spanish thriller El Cuerpo (Spanish title) also known as \"The Body\", and features Arjun Sarja, Shaam and Manisha Koirala in the lead roles.', ' With music composed by Ilayaraaja, the film was simultaneously shot in Kannada and Tamil; the former released first on February 26, 2016 while the later released on July 1, 2016.', ' The film was dubbed and released in Telugu as \"Notuku Potu\" in 2017.']], ['Toys in the Attic (2009 film)', ['Toys in the Attic (Czech: Na půdě aneb Kdo má dneska narozeniny?', ' ; festival title: In the Attic: Who Has a Birthday Today?)', ' is a 2009 Czech-French-Japanese-Slovak primarily stop-motion animated fantasy comedy thriller family film directed by Jiří Barta and written by Edgar Dutka and Barta which depicts a community of toys and other objects in an attic who come to life when no human is around.', ' It is an international co-production of Czech, Japanese and Slovak companies.', ' The film was released first in the Czech Republic on 5 March 2009 and has been shown subtitled at film festivals internationally.', \" An American dub – adapted, produced and directed by Vivian Schilling and performed by actors including Forest Whitaker, Joan Cusack, Cary Elwes and Schilling herself – has been recorded, which the film was first shown with on 3 March 2012 at the New York International Children's Film Festival and was released nationally on 24 August 2012 by Hannover House.\"]], ['Deewana (1992 film)', [\"Deewana (English: 'Crazy' ) is a 1992 Indian romantic drama film directed by Raj Kanwar, and produced by Guddu Dhanoa and Lalit Kapoor and featuring Shah Rukh Khan, Divya Bharti and Rishi Kapoor in the lead.\", \" This was Shah Rukh's debut release, and he appears only in the second half of the film.\", ' He replaced Armaan Kohli, who walked out of the project due to creative differences after the first schedule.', ' The film released on June 25, 1992. \"', 'Dil Aashna Hai\" was supposed to be the debut movie of Shahrukh Khan however \"Deewana\" was released first.']], ['Christopher McLeod', [\"Christopher (Toby) McLeod is the project director of Earth Island Institute's Sacred Land Film Project, which he founded in 1984 as one of Earth Island's original projects.\", ' Since 2006 he has been producing and directing the four-part documentary film series \"Standing on Sacred Ground\", which premiered in 2013 at the Mill Valley Film Festival and aired nationally on PBS in 2015.', ' \"Standing on Sacred Ground\" features eight indigenous communities around the world fighting to protect their sacred places.', ' The award-winning series visits Altaians in Russia, the Winnemem Wintu in northern California, Papua New Guinea, the tar sands of Canada, the Gamo Highlands of Ethiopia, Peru, Australia and Hawaii.', ' McLeod produced and directed the award-winning documentary \"In the Light of Reverence\" (2001) and has made three other award-winning documentary films: \"The Four Corners: A National Sacrifice Area?\"', ' (1983) with Glenn Switkes and Randy Hayes, (Winner of the Student Academy Award).', ' \"Downwind/Downstream\" (1988) with Robert Lewis, and \"NOVA: Poison in the Rockies\" (1990).', ' His first film was the 9-minute short \"The Cracking of Glen Canyon Damn—with Edward Abbey and Earth First!\"', ' (1982) with Glenn Switkes and Randy Hayes.', \" The focus of these educational projects has been to increase public awareness and understanding of sacred natural sites, indigenous peoples' cultural practices and worldviews, and environmental justice.\"]], ['Snegithiye', ['Snegithiye (English:Oh Friend! \"', '(female)\" ) is a 2000 Tamil mystery thriller film directed by Priyadarshan.', ' The story is loosely based on the 1999 Marathi film \"Bindhaast\" written by Chandrakant Kulkarni.', ' The film notably features only female characters in the lead roles, played by Jyothika, Sharbani Mukherjee, Tabu and Ishita Arun.', ' Music was composed by Vidyasagar.', ' The film, released in 2000, proved to be an average grosser at the box office but bagged positive reviews from critics.', ' Today, it is considered a cult classic that was underrated at the time of its release.', ' Originally planned to be made as a bilingual, in Tamil and in Malayalam, the film released first in Tamil only, while the Malayalam dubbed version, \"Raakilipattu\", as well as the dubbed Hindi version, \"Friendship\", released seven years later.']], ['Tar Creek (film)', ['Tar Creek is a 2009 feature-length environmental documentary about the Tar Creek Superfund Site, which at one time was considered the worst environmental disaster in the United States.', \" It was directed Matt Myers, who also wrote the film's script and served as its narrator.\"]], ['Secrets of Life', ['Secrets of Life is a 1956 American documentary film written and directed by James Algar.', ' The documentary follows the changing world of nature, the sky, the sea, the sun, planets, insects and volcanic action.', ' The documentary was released on November 6, 1956, by Buena Vista Distribution.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.737\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5addf6135542990dbb2f7f23', 'answer': 'Genderqueer', 'question': 'Which identifier: transwomen, cis women,or genderqueer, is a combination of masculinity and femininity or neither in gender expression? ', 'supporting_facts': [['Discwoman', 0], ['Genderqueer', 1]], 'context': [['Transgender', ['Transgender people are people who have a gender identity or gender expression that differs from their assigned sex.', ' Transgender people are sometimes called \"transsexual\" if they desire medical assistance to transition from one sex to another.', ' \"Transgender\" is also an umbrella term: in addition to including people whose gender identity is the \"opposite\" of their assigned sex (trans men and trans women), it may include people who are not exclusively masculine or feminine (people who are genderqueer/non-binary, e.g. bigender, pangender, genderfluid, or agender).', ' Other definitions of \"transgender\" also include people who belong to a third gender, or conceptualize transgender people as a third gender.', ' Infrequently, the term \"transgender\" is defined very broadly to include cross-dressers, regardless of their gender identity.']], ['Genderqueer', ['Genderqueer (GQ), also termed non-binary (NB), is a catch-all category for gender identities that are not exclusively masculine or feminineidentities which are thus outside the gender binary and cisnormativity.', ' Genderqueer people may express a combination of masculinity and femininity, or neither, in their gender expression.']], ['Transitioning (transgender)', [\"Transitioning is the process of changing one's gender presentation and/or sex characteristics to accord with one's internal sense of gender identity – the idea of what it means to be a man or a woman, or genderqueer (in-between).\", ' For transgender and transsexual people, this process commonly involves reassignment therapy (which may include hormone replacement therapy and sex reassignment surgery), with their gender identity being opposite that of their birth-assigned sex and gender.', ' Transitioning might involve medical treatment, but it does not always involve it.', ' For genderqueer people, it is neither solely female nor male.', ' Cross-dressers, drag queens, and drag kings tend not to transition, since their variant gender presentations are (usually) only adopted temporarily.']], ['Soft butch', ['A soft butch, or stem (stud-fem), is a woman who exhibits some stereotypical butch and lesbian traits without fitting the masculine stereotype associated with butch lesbians.', ' Soft butch is on the spectrum of butch, as are stone butch and masculine, whereas on the contrary, ultra fem, high femme, and lipstick lesbian are some labels on the spectrum of lesbians with a more prominent expression of femininity, also known as femmes.', ' Soft butches have gender identities of women, but primarily display masculine characteristics; soft butches predominantly express masculinity with a touch of femininity.', ' The \"hardness\", or label depicting one\\'s level of masculine expression as a butch is dependent upon the fluidity of her gender expression.', ' Soft butches might want to express themselves through their clothing and hairstyle in a more masculine way, but their behavior in a more traditionally feminine way.', ' For example, these traits of a soft butch may or may not include short hair, clothing that was designed for men, and masculine mannerisms and behaviors.', ' Soft butches generally appear androgynous, rather than adhering to strictly feminine or masculine norms and gender identities.', ' Soft butches generally physically, sexually, and romantically express themselves in more masculine than feminine ways in the majority of those categories.']], ['Femininity', ['Femininity (also called girlishness, womanliness or womanhood) is a set of attributes, behaviors, and roles generally associated with girls and women.', ' Femininity is socially constructed, but made up of both socially-defined and biologically-created factors.', ' This makes it distinct from the definition of the biological female sex, as both males and females can exhibit feminine traits.', ' People who exhibit a combination of both masculine and feminine characteristics are considered androgynous, and feminist philosophers have argued that gender ambiguity may blur gender classification.', ' Modern conceptualizations of femininity also rely not just upon social constructions, but upon the individualized choices made by women.']], ['Gender identity', [\"Gender identity is one's personal experience of one's own gender.\", ' Gender identity can correlate with assigned sex at birth, or can differ from it completely.', \" All societies have a set of gender categories that can serve as the basis of the formation of a person's social identity in relation to other members of society.\", ' In most societies, there is a basic division between gender attributes assigned to males and females, a gender binary to which most people adhere and which includes expectations of masculinity and femininity in all aspects of sex and gender: biological sex, gender identity, and gender expression.', ' In all societies, some individuals do not identify with some (or all) of the aspects of gender that are assigned to their biological sex; some of those individuals are transgender or genderqueer.', ' Some societies have third gender categories.']], ['Gender variance', ['Gender variance, or gender nonconformity, is behavior or gender expression by an individual that does not match masculine and feminine gender norms.', ' People who exhibit gender variance may be called \"gender variant\", \"gender non-conforming\", \"gender diverse,\" \"gender atypical\" or \"genderqueer\", and may be transgender or otherwise variant in their gender identity.', ' In the case of transgender people, they may be perceived, or perceive themselves as, gender nonconforming before transitioning, but might not be perceived as such after transitioning.', ' Some intersex people may also exhibit gender variance.']], ['Gender polarization', ['Gender polarization is a concept in sociology by American psychologist Sandra Bem which states that societies tend to define femininity and masculinity as polar opposite genders, such that male-acceptable behaviors and attitudes are not seen as appropriate for women, and vice versa.', ' The theory is an extension of the sex and gender distinction in sociology in which sex refers to the biological differences between men and women, while gender refers to the cultural differences between them, such that \"gender\" describes the \"socially constructed roles, behaviours, activities, and attributes that a given society considers appropriate for men and women\".', ' According to Bem, gender polarization begins when natural sex differences are exaggerated in culture; for example, women have less hair than men, and men have more muscles than women, but these physical differences are exaggerated culturally when women remove hair from their faces and legs and armpits, and when men engage in body building exercises to emphasize their muscle mass.', ' She explained that gender polarization goes further, when cultures construct \"differences from scratch to make the sexes even more different from one another than they would otherwise be\", perhaps by dictating specific hair styles for men and women, which are noticeably distinct, or separate clothing styles for men and women.', ' When genders become polarized, according to the theory, there is no overlap, no shared behaviors or attitudes between men and women; rather, they are distinctly opposite.', ' She argued that these distinctions become so \"all-encompassing\" that they \"pervade virtually every aspect of human existence\", not just hairstyles and clothing but how men and women express emotion and experience sexual desire.', ' She argued that male-female differences are \"superimposed on so many aspects of the social world that a cultural connection is thereby forged between sex and virtually every other aspect of human experience\".']], ['Discwoman', ['Discwoman is a New York based collective, booking agency, and event platform representing and showcasing female-identified (cis women, transwomen, and gender queer) talent in the electronic music community.', ' It was founded in 2014 by Frankie Decaiza Hutchinson who does the outreach for the agency dealing with Public Relations and social media, Emma Burgess-Olson (a.k.a. UMFANG) as the resident DJ, and Christine McCharen-Tran who is the event producer and business powerhouse.', \" Discwoman's regular club nights and touring events highlight emerging and established artists from around the world.\", ' Music produced by world-renowned female artists include The Black Madonna, Nicole Moudaber, Star Eyes, Sandunes, Demian Licht, and Nina Sonik whom have contributed to the electronic music culture.', ' The gender imbalance in EDM (electronic dance music) is self-evident showing women making up to ~10.8% of artists in electronic music festivals.', ' In a 2015 report by , it is stated that men comprised 82% of 44 international festivals’ lineups.', ' Discwoman gives feminine-identified talent the platform and more visibility by booking them at bigger venues, streamlining the growth process, and ensuring the artists they are paid what they are worth in a male-dominated dance music industry.']], ['Transmisogyny', ['Transmisogyny (sometimes trans-misogyny) is the intersection of transphobia and misogyny.', ' Transphobia is defined as \"the irrational fear of, aversion to, or discrimination against transgender or transsexual people\".', ' Misogyny is defined as \"a hatred of women\".', ' Therefore, transmisogyny includes negative attitudes, hate, and discrimination of transgender or transsexual individuals who fall on the feminine side of the gender spectrum.', ' The term was coined by Julia Serano in her 2007 book \"Whipping Girl\" and used to describe the unique discrimination faced by trans women because of \"the assumption that femaleness and femininity are inferior to, and exist primarily for the benefit of, maleness and masculinity\", and the way that transphobia intensifies the misogyny faced by trans women (and vice versa).', ' The term discusses how many trans women experience an additional layer of misogyny in the form of fetishization; Serano talks about how society views trans women in certain ways that sexualize them, such as them transitioning for sexual reasons, or ways where they’re seen as sexually promiscuous.Transmisogyny is a central concept in transfeminism and is commonly seen in intersectional feminist theory.', \" The suggestion that trans women's femaleness (rather than their femininity) is a source of transmisogyny is rejected by some feminists, who do not regard trans women as female.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.737\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ade79335542997c77adee38', 'answer': 'Ohio River', 'question': 'The Ran Paul presidential campaign, 2016 event was held at a hotel on what river?', 'supporting_facts': [['Rand Paul presidential campaign, 2016', 0], ['Galt House', 2]], 'context': [['NGP VAN', ['NGP VAN is a privately owned American company specializing in helping progressive campaigns and organizations leverage technology to meet their goals.', ' In 2009, the company was the largest partisan provider of campaign compliance software, used by most Democratic members of Congress.', \" The company's services have been utilized by clients such as the Obama 2008 presidential campaign, the Obama 2012 presidential campaign, the Bernie Sanders 2016 presidential campaign, the Hillary Clinton 2016 presidential campaign, the British Liberal Democrats, and the Liberal Party of Canada.\", ' Its current president and CEO, Stuart Trevelyan, was a veteran of the 1992 Clinton-Gore \"War Room\", providing research, analysis, and whip counts to the Clinton Administration as a member of the White House Office of Legislative Affairs.']], ['Icebreaker at The Granite', ['The Spider Performance Icebreaker At The Granite is an annual cashspiel, or curling tournament, that takes place at the Granite Curling Club in Winnipeg.', ' The Event has been a part of the World Curling Tour since 2017.', ' The tournament is held in a round robin format.', ' The 2016 event was known as the 2016 Performance Spider Midweek Special and was held in Waterloo, Ontario and was only part of the Ontario Curling Tour.', \" The 2016 event included both genders, and in 2017 a separate women's event was added.\"]], ['Galt House', ['The Galt House is a 25-story, 1300-room hotel in Louisville, Kentucky established in 1972.', ' It is named for a nearby historic hotel erected in 1837 and demolished in 1921.', \" The Galt House is the city's only hotel on the Ohio River.\"]], ['T in the Park', ['T in the Park festival was a major Scottish music festival that had been held annually from 1994 to 2016.', ' It is named after its main sponsor, the brewing company Tennents.', ' The event was originally held at Strathclyde Park, Lanarkshire but was held at the disused Balado airfield, Kinross-shire from 1997 to 2014.', ' In 2015 the festival moved to Strathallan Castle, Strathallan, Perthshire.', ' Originally a two-day event, the festival became a three-day event in 2007.', ' Promoted by DF Concerts, the event attracts up to 255,000 people, along with 70,000 campers.', ' The future of the festival is uncertain.', ' The 2017 event was cancelled due to problems at the 2016 event.', ' It was officially replaced with the TRNSMT festival which takes place on the same weekend at Glasgow Green.', ' The 2017 event was a success with a follow-up for 2018 in the pipeline, meaning that T in the Park is \"not looking likely\" for 2018.']], ['Ron Paul presidential campaign, 1988', ['The Ron Paul presidential campaign of 1988 began in early 1987 when former Congressman Ron Paul of Texas announced his candidacy for the 1988 presidential nomination of the Libertarian Party.', \" He joined the third party after leaving the Republican Party over the Reagan administration's handling of the federal budget.\", ' He ran on a platform that included non-interventionism in foreign conflicts, decriminalization of illegal drugs on a federal level, a return to the gold standard, the abolition of the Federal Reserve and a reduction in all government spending.']], ['Donald Trump presidential campaign, 2000', ['Donald Trump\\'s presidential campaign of 2000 for the nomination of the Reform Party began when real estate magnate Donald Trump of New York announced the creation of a presidential exploratory committee on the October 7, 1999 edition of \"Larry King Live\".', ' Though Trump had never held elected office, he was well known for his frequent comments on public affairs and business exploits as head of The Trump Organization.', ' He had previously considered a presidential run in 1988 as a Republican, but chose not to run.', ' For 2000, Minnesota Governor Jesse Ventura persuaded Trump to seek the presidential nomination of the Reform Party, which was fracturing despite achieving ballot access and qualifying for matching funds as a result of the 1996 presidential campaign of businessman Ross Perot.', ' Trump\\'s entrance into the Reform Party race coincided with that of paleoconservative commentator Pat Buchanan, whom Trump attacked throughout the campaign as a \"Hitler-lover.\"']], ['Liberty Movement', ['Liberty movement may also refer to the movement in America popularized by Ron Paul (Ron Paul presidential campaign, 2008).']], ['Louisville Zombie Attack', ['The Louisville Zombie Attack started on August 29, 2005 as a flash mob, is a public event in Louisville, Kentucky held annually in August.', ' Before 2016, it had been held on August 29 at 8:29 p.m., but the 2016 event was moved to August 27 (which fell on a Saturday that year) at the same time and rebranded as the \"Louisville Zombie Walk\".', \" This resulted in a lawsuit between two of the event's co-founders, John King and Lyndi Curtis, which resulted in Curtis abandoning all rights to the event.\", \" The Louisville Zombie Walk's co-organizer, Jason Bessemann, was not named in the suit and went on to promote the 2017 Louisville Zombie Walk as a separate event three days before the original.\", ' Several thousand participants dressed and made up as zombies gather in the Highlands area and march down Bardstown Road to the end point of the walk.', ' An after party is typically thrown by all the bars included in the nightlife throughout the street after the walk.', \" Some of the bars included are: Nowhere Bar, Highlands Taproom, Big Bar, Baxter's, and many others in the local area.\", ' Throughout the years, the walk grew by thousands as word of mouth caught on.', ' The last few years, the walk has generated over 10,000 walkers, with 30,000 in 2013 and 32,000 in 2014.', ' Over 40,000 walkers were anticipated for the 2015 event.', ' It began as a birthday party for three friends, but has turned into a real event recognized by local businesses.', ' The event typically costs around $10,000 to function, but the event is kept free to the public due to donations.']], ['Rand Paul presidential campaign, 2016', ['The 2016 presidential campaign of Rand Paul, the junior United States Senator from Kentucky, was announced on April 7, 2015 at an event at the Galt House in Louisville, Kentucky.', \" First elected to the U.S. Senate in the 2010 election, Paul's candidacy for the Republican nomination for President of the United States in 2016 had been widely speculated since early 2013.\"]], ['Tony Fabrizio', ['Anthony Fabrizio (born 1960) is an American Republican pollster and strategist.', \" He is the principal in Fabrizio, Lee & Associates, and was the pollster for Donald Trump's fall 2016 Presidential campaign, former Senator Bob Dole's 1996 Presidential campaign, U.S. Senator Rand Paul's U.S. Senate and 2016 Presidential campaign, and former Governor Rick Perry's 2012 Presidential campaign, among others.\", ' He also served as a pollster for the U.S. Chamber of Commerce in the 2014 midterm elections.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.738\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adcf2b55542992c1e3a24cd', 'answer': 'sixteen', 'question': \"How many children's books has the writer of the sitcom Maid Marian and her Merry Men written ? \", 'supporting_facts': [['Maid Marian and Her Merry Men', 0], ['Tony Robinson', 3]], 'context': [['Ramsay Gilderdale', ['Ramsay Wilson Gilderdale (born 5 August 1962 in Wakefield, West Yorkshire) is an English actor, who played Guy of Gisbourne in BBC comedy \"Maid Marian and Her Merry Men\".', ' He also appeared in \"Rumpole of the Bailey\" and \"Blackadder\\'s Christmas Carol\".']], ['Howard Lew Lewis', ['Howard Lew Lewis (born 21 August 1941) is an English comedian and actor, best known for his roles in comedy series including \"Maid Marian and her Merry Men\" and \"Brush Strokes\".']], ['Adam Morris', ['Adam Morris (also known as Wayne Morris) is a British stage and screen actor whose most notable roles have been Robin Hood (or Robin of Kensington) in the television comedy series \"Maid Marian and Her Merry Men\", and more recently, Philip Norton in \"Genie in the House\".', ' Trained at London\\'s Central School of Speech and Drama, his stage appearances include Bri in \"A Day in the Death of Joe Egg\" and Gordon in \"The Throne\" for the New Vic, and he appeared for one week in \"Speed the Plow\" at the Playhouse Theatre, London, opposite Lindsay Lohan while Richard Schiff was indisposed.']], ['Kate Lonergan', ['Helen Catherine \"Kate\" Lonergan (born 4 January 1962 in Barton-upon-Irwell, Lancashire) is an English former actress, best known for playing the role of Marian in the 1989–94 BBC One children\\'s television series \"Maid Marian and her Merry Men\".']], ['David Lloyd (actor)', ['David Lloyd (born 17 May 1955) is an English actor and screenwriter, perhaps best known from his role in \"Maid Marian and her Merry Men\", where he played Graeme, one of the two guards (alongside Mark Billingham\\'s Gary).']], ['Mike Edmonds', ['Mike Edmonds (born 13 January 1944) is an English actor with dwarfism, known for his role as Little Ron in the children\\'s television show \"Maid Marian and Her Merry Men\".']], ['Robin Hood: The Legend of Sherwood', ['Robin Hood: The Legend of Sherwood is a 2002 stealth-based real-time tactics video game developed by Spellbound Studios.', ' It is similar to games such as \"\" and the \"Commandos\" series.', ' In the game, the player controls up to five characters in a setting based on the stories of the protagonist, Robin Hood.', \" The player can also control Robin Hood's Merry Men, including Little John, Friar Tuck, Will Scarlet, Will Stutely, and Maid Marian.\", \" Robin and his crew must evade the cruel Sheriff of Nottingham and his henchmen and stop the machinations of the vile usurper to England's throne, Prince John.\", ' However, Robin must avoid killing enemies as much as possible, or he will not be able to recruit as many new Merry Men.']], ['Tony Robinson', ['Sir Anthony \"Tony\" Robinson (born 15 August 1946) is an English actor, comedian, author, presenter, historian and political activist.', ' He is known for playing Baldrick in the BBC television series \"Blackadder\" and for hosting the Channel 4 programmes \"Time Team\" and \"The Worst Jobs in History\".', ' Robinson is a member of the Labour Party and has served on its National Executive Committee.', \" He has also written sixteen children's books.\"]], ['Maid Marian and Her Merry Men', [\"Maid Marian and her Merry Men is a British children's sitcom created and written by Tony Robinson and directed by David Bell.\", ' It began in 1989 on BBC One and ran for four series, with the last episode shown in 1994.', ' The show was a partially musical comic retelling of the legend of Robin Hood, placing Maid Marian in the role of leader of the Merry Men, and reducing Robin to an incompetent ex-tailor.']], ['List of Maid Marian and Her Merry Men episodes', ['The following is a list of the episodes of the BBC television series \"Maid Marian and her Merry Men\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.739\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a89d8d055429946c8d6e9df', 'answer': 'Lady Frederick Windsor', 'question': 'ElvenQuest is a comic fantasy broadcast that stars what English actress who is the wife of Lord Frederick Windsor, and who is formally referred to as what?', 'supporting_facts': [['ElvenQuest', 0], ['Sophie Winkleman', 0], ['Sophie Winkleman', 1]], 'context': [['Comic fantasy', ['Comic fantasy is a subgenre of fantasy that is primarily humorous in intent and tone.', ' Usually set in imaginary worlds, comic fantasy often includes puns on and parodies of other works of fantasy.', ' It is sometimes known as low fantasy in contrast to high fantasy, which is primarily serious in intent and tone.', ' The term \"low fantasy\" is used to represent other types of fantasy, however, so while comic fantasies may also correctly be classified as low fantasy, many examples of low fantasy are not comic in nature.']], ['ElvenQuest', ['ElvenQuest is a comic fantasy broadcast on BBC Radio 4 by Anil Gupta and Richard Pinto, and starring Stephen Mangan, Alistair McGowan, Darren Boyd, Kevin Eldon, Sophie Winkleman and Dave Lamb.', ' The series takes place in the world of Lower Earth, a parody of Middle-earth from \"The Lord of the Rings\" by J. R. R. Tolkien.', ' In Lower Earth, a band of warriors go forth to search for a mythical sword to save Lower Earth from the evil Lord Darkness (played by McGowan).', ' In order to do so, they must find \"The Chosen One\" who will save Lower Earth.', ' The Chosen One is Amis, a dog in the real world which belongs to a fantasy novelist called Sam Porter (played by Mangan).', ' The first series was broadcast from 29 March to 3 June 2009 and the second from 18 November 2010.', ' The third series began broadcasting from 17 October 2011.', ' The fourth series began broadcasting 12 February 2013.']], ['ABS-CBN (TV network)', [\"ABS-CBN (an initialism of the network's former names, Alto Broadcasting System - Chronicle Broadcasting Network) is a Filipino commercial broadcast television network that is the flagship property of ABS-CBN Corporation, a company under Lopez Group.\", ' The network is headquartered at the ABS-CBN Broadcasting Center in Quezon City, with additional offices and production facilities in 25 major cities including Baguio, Naga, Iloilo, Cebu, and Davao.', ' ABS-CBN is formally referred to as \"The \"Kapamilya\" Network\", \"Kapamilya\" is a Filipino term which means a member of a family, is originally introduced in 1999 and was officially introduced in 2003 during the celebration of the 50th year anniversary of Philippine television.', ' It is the largest television network in the country in terms of revenues, assets, and international coverage.']], ['List of British regional nicknames', ['In addition to formal demonyms, many nicknames are in common use for residents of the different countries, regions and places of the United Kingdom.', ' For example, residents of Liverpool, formally referred to as \"Liverpudlians\", are also referred to by the nickname \"Scousers\".', ' Some nicknames are a badge of pride; in other cases they may be regarded as offensive.', ' Many of the names listed below are merely the nicknames of local football teams and are rarely, if ever, used in a non-football context.']], ['Lord Frederick Windsor', ['Lord Frederick Windsor (Frederick Michael George David Louis; born 6 April 1979), also nicknamed Freddie Windsor, is a British financial analyst, and the only son of Prince and Princess Michael of Kent.']], ['Sophie Winkleman', ['Sophie Lara Winkleman (born 5 August 1980) is an English actress.', \" As the wife of Lord Frederick Windsor, son of Queen Elizabeth II's first cousin Prince Michael of Kent, she is formally referred to as Lady Frederick Windsor, but continues to use her maiden name in her professional career.\"]], ['Lord Frederick Cambridge', ['Lord Frederick Cambridge (\"Frederick Charles Edward\") (born Prince Frederick of Teck) (24 September 1907 – 15 May 1940) was a descendant of the British Royal Family.', ' He was the younger son of the Adolphus Cambridge, 1st Marquess of Cambridge, formerly the Duke of Teck, and a nephew of Queen Mary, the consort of King George V.']], ['Fred Warren', [\"Frederick Windsor 'Freddie' Warren (23 December 1907 – 1986) was a Welsh professional footballer and Wales international.\"]], ['Zeynab Javadli', ['Zeynab Javadli (Azeri: \"Zeynəb Cavadlı\"), formally referred to as Sheikha Zaynab (Arabic: الشيخة زينب\\u200e \\u200e ) (born 19 July 1991) is an Azerbaijani former gymnast and a wife of Emirati royal Saeed bin Maktoum bin Rashid Al Maktoum.']], ['The Eye of Tandyla', ['\"The Eye of Tandyla\" is a fantasy story by American writer L. Sprague de Camp, part of his Pusadian series.', ' It was first published in the magazine \"Fantastic Adventures\" for May, 1951, and first appeared in book form in de Camp\\'s collection \"The Tritonian Ring and Other Pusadian Tales\" (Twayne, 1953).', ' The story has also appeared in the magazine \"Fantastic\" for November 1965, the anthologies \"Time Untamed\" (1967), \"The Magic of Atlantis\" (1970), \"Wizards\" (1983), and \"The Mammoth Book of Seriously Comic Fantasy\" (1999) (also published as \"The Mammoth Book of Comic Fantasy II\"), and the de Camp omnibus collection \"Lest Darkness Fall/Rogue Queen/The Tritonian Ring and Other Pusadian Tales\" (2014).', ' It has also been translated into French, Spanish, Italian, German and Russian.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.739\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae1b2b9554299422ee99684', 'answer': 'Robert Digges Wimberly Connor', 'question': 'Who did President Franklin Roosevelt appoint that was responsible to transmit votes of the Electoral College to Congress?', 'supporting_facts': [['Robert Digges Wimberly Connor', 0], ['National Archives and Records Administration', 2]], 'context': [['Wendell Willkie', ['Wendell Lewis Willkie (born Lewis Wendell Willkie; February 18, 1892 – October 8, 1944) was an American lawyer and corporate executive, and the 1940 Republican nominee for President.', \" Willkie appealed to many convention delegates as the Republican field's only interventionist: although the U.S. remained neutral prior to Pearl Harbor, he favored greater U.S. involvement in World War II to support Britain and other Allies.\", ' His Democratic opponent, incumbent President Franklin D. Roosevelt, won the 1940 election with about 55% of the popular vote and took the electoral college vote by a wide margin.']], ['James Farley', ['James Aloysius \"Jim\" Farley (May 30, 1888 – June 9, 1976) was one of the first Irish Catholic politicians in American history to achieve success on a national level.', ' He simultaneously served as Chairman of the New York State Democratic Committee, Chairman of the Democratic National Committee, and Postmaster General under the first two administrations of President Franklin Roosevelt.', \" A business executive and dignitary and a Knight of Malta, Farley was commonly referred to as a political kingmaker, and he was responsible for Roosevelt's rise to the presidency.\", \" Farley was the campaign manager for New York State politician Alfred E. Smith's 1922 gubernatorial campaign and Roosevelt's 1928 and 1930 gubernatorial campaigns as well as Roosevelt's presidential campaigns of 1932 and 1936.\", ' Farley predicted large landslides in both, and revolutionized the use of polling data.']], ['Robert Digges Wimberly Connor', ['Robert Digges Wimberly Connor (September 26, 1878 – February 25, 1950) was an American historian and the first Archivist of the United States, 1934-1941.', ' He was born to Henry G. Connor and Kate Whitfield Connor on September 26, 1878, in Wilson, North Carolina.', ' At the time that President Franklin Roosevelt appointed him to head the National Archives, Connor was serving as a professor of history at the University of North Carolina at Chapel Hill, from which he graduated himself in 1899.', ' Connor served as the third president of the Society of American Archivists between 1941-1943.']], ['Roosevelt Franklin', ['Roosevelt Franklin was a Muppet featured on the children\\'s television series \"Sesame Street\" during the early 1970s.', ' He is purple with shaggy black hair that stands on end.', ' His name is a word play on the name of the late US President Franklin Roosevelt, but the first and last names are reversed.', ' \"Sesame Street\" cast member Matt Robinson, who also played Gordon on the series for the first two seasons (1969-71), provided Roosevelt Franklin\\'s voice.']], ['Emergency Banking Act', ['The Emergency Banking Act (the official title of which was the Emergency Banking Relief Act), Public Law 1, 48 Stat.', ' 1 (March 9, 1933), was an act passed by the United States Congress in March 1933 in an attempt to stabilize the banking system.', ' Beginning on February 14, 1933, Michigan, an industrial state which had been hit particularly hard by the Great Depression in the United States, declared an eight-day bank holiday.', ' Fears of other bank closures spread from state to state as people rushed to withdraw their deposits while they still could do so.', ' Within weeks, all other states held their own bank holidays in an attempt to stem the bank runs (on March 4th, Delaware became the 48th and last state to close its banks.)', \" Following his inauguration on March 4, 1933, President Franklin Roosevelt set out to rebuild confidence in the nation's banking system.\", ' On March 6 he declared a four-day \"national\" banking holiday that kept all banks shut until Congress could act.', \" A draft law prepared by the Treasury staff during Herbert Hoover's administration, was passed on March 9, 1933.\", ' The new law allowed the twelve Federal Reserve Banks to issue additional currency on good assets so that banks that reopened would be able to meet every legitimate call.']], ['United States presidential election', ['The election of President and Vice President of the United States is an indirect election in which citizens of the United States who are registered to vote in one of the 50 U.S. states or Washington, D.C. cast ballots for members of the U.S. Electoral College, known as electors.', ' These electors then in turn cast direct votes, known as electoral votes, for President and Vice President.', ' The candidate who receives an absolute majority of electoral votes for President or Vice President (currently, at least 270 out of a total of 538) is then elected to that office.', ' If no candidate receives an absolute majority for President, the House of Representatives chooses the President; if no one receives a majority for Vice President, then the Senate chooses the Vice President.']], ['National Archives and Records Administration', ['The National Archives and Records Administration (NARA) is an independent agency of the United States government charged with preserving and documenting government and historical records and with increasing public access to those documents, which comprise the National Archives.', ' NARA is officially responsible for maintaining and publishing the legally authentic and authoritative copies of acts of Congress, presidential proclamations and executive orders, and federal regulations.', ' The NARA also transmits votes of the Electoral College to Congress.']], ['Franklin Delano Roosevelt Foundation', ['The Franklin Delano Roosevelt Foundation is a private 501(c)3 US public charity based at Adams House, Harvard University.', ' Founded as the FDR Suite Foundation in 2008, its original goal was to restore the Harvard rooms of Franklin Roosevelt, the 32nd President of the United States.', ' The Foundation adopted its current name in 2014 to better reflect its broadened philanthropic mission to promote and preserve the legacy of Franklin Roosevelt throughout the world.', ' The Foundation currently comprises three principal initiatives:']], ['United States presidential election, 1792', ['The United States presidential election of 1792 was the second quadrennial presidential election.', ' It was held from Friday, November 2 to Wednesday, December 5, 1792.', ' Incumbent President George Washington was elected to a second term by a unanimous vote in the electoral college.', ' As in the first presidential election, Washington is considered to have run unopposed.', ' Electoral rules of the time, however, required each presidential elector to cast two votes without distinguishing which was for president and which for vice president.', ' The recipient of the most votes would then become president, and the runner-up vice president.', ' Incumbent Vice President John Adams received 77 votes and was also re-elected (Washington received 132 votes, or one from each elector).', ' This election saw the least amount of popular votes elect the winner in American history.']], ['Harry S. Truman', ['Harry S. Truman (May 8, 1884December 26, 1972) was the 33rd President of the United States (1945–53), assuming that office upon the death of Franklin D. Roosevelt during the waning months of World War II.', ' He is known for launching the Marshall Plan to rebuild the economy of Western Europe, for leading the Cold War against Soviet and Chinese communism by establishing the Truman Doctrine and NATO, and for intervening in the Korean War.', \" In domestic affairs, he was a moderate Democrat whose liberal proposals were a continuation of Franklin Roosevelt's New Deal, but the conservative-dominated Congress blocked most of them.\", ' He used the veto power 180 times, more than any president since then, and saw 12 overridden by Congress; only Grover Cleveland and Franklin D. Roosevelt used the veto so often, and only Gerald Ford and Andrew Johnson saw so many veto overrides.', ' He is also the only world leader to have ever used nuclear weapons in war, desegregated the U.S. Armed Forces, supported a newly independent Israel, and was a founder of the United Nations.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.740\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abd09585542996e802b469b', 'answer': '2011', 'question': 'What year did the musical premiere that was based on the 2007 Irish romantic musical film of the same name?', 'supporting_facts': [['Once (musical)', 0], ['Once (musical)', 3], ['Once (film)', 0]], 'context': [['Show Boat (1936 film)', ['Show Boat is a 1936 romantic musical film directed by James Whale, based on the musical of the same name by Jerome Kern and Oscar Hammerstein II, which in turn was adapted from the novel of the same name by Edna Ferber.']], ['Once (musical)', ['Once is a musical based on the 2007 film of the same name by John Carney.', ' Like the film, music and lyrics were by Glen Hansard and Markéta Irglová, including the Academy Award-winning \"Falling Slowly\".', ' The book for the musical was written by Enda Walsh.', ' The musical premiered at the New York Theatre Workshop in 2011, before transferring to Broadway in 2012.', ' The production received eleven 2012 Tony Award nominations, and won eight including Best Musical, Best Actor and Best Book.', ' The musical also won the 2012 Drama Desk Award for Outstanding Musical and the 2013 Grammy Award for Best Musical Theater Album.', ' It has since spawned a London production, with a North American Tour which started on 1 October 2013.']], ['Adaraneeya Kathawak', ['Adaraneeya Kathawak (English: \"A Melody Of Love\" ) is a 2016 Sri Lankan Sinhala romantic musical film, directed and co-produced by Priyantha Colombage.', ' It was released on 10 June 2016.', ' Starring Hemal Ranasinghe, Udari Warnakulasooriya, Bimal Jayakody and Aruni Rajapaksha in the lead roles.', ' The film has influenced by romantic musical Bollywood film \"Aashiqui 2\".']], ['South Pacific (1958 film)', ['South Pacific is a 1958 American romantic musical film based on the Rodgers and Hammerstein musical \"South Pacific\", which in turn based on James A. Michener\\'s short-story collection \"Tales of the South Pacific\".', ' The film, directed by Joshua Logan, starred Rossano Brazzi, Mitzi Gaynor, John Kerr and Ray Walston in the leading roles with Juanita Hall as Bloody Mary, the part that she had played in the original stage production.', ' The film was nominated for three Academy Awards, winning the Academy Award for Best Sound for Fred Hynes.']], ['Kings (2007 film)', ['Kings is a 2007 Irish film written & directed by Tom Collins and based on Jimmy Murphy\\'s play \"The Kings of the Kilburn High Road\".', ' The film is bilingual, having both Irish and English dialogues.', \" It premiered at the Taormina Film Festival (Italy) in June 2007, and was selected as Ireland's official entry for the 2008 Academy Awards in the best foreign-language film category.\", ' The film tells the story of a group of Irish friends who, after emigrating to England 30 years previously, meet for the funeral of a friend.', ' In 2008, the Irish postal service, An Post, issued a series of stamps honouring the Irish film industry.', ' Colm Meaney, as Joe Mullan, was featured on the 55 cent stamp.']], [\"God's Gift to Women\", [\"God's Gift to Women (1931) is an American Pre-Code romantic musical comedy film by Warner Brothers.\", ' The film starred Frank Fay, Charles Winninger, Laura LaPlante, Louise Brooks and Joan Blondell.', ' The film was based on the play called \"The Devil Was Sick\" by Jane Hinton, and was originally completed as a musical film.', ' Due to audience distaste for musicals, however, all the songs were cut in American prints.', ' The complete film was released intact in other countries, where there was no such decline in popularity.']], ['Borstal Boy (film)', ['Borstal Boy is a 2000 British/Irish romantic drama film adaptation of the Brendan Behan autobiographical novel of the same name.', ' The film is written and directed by Irish playwright Peter Sheridan.']], ['Once (film)', ['Once is a 2007 Irish romantic musical film written and directed by John Carney.', ' The film stars Glen Hansard and Markéta Irglová as two struggling musicians in Dublin, Ireland.', \" Hansard and Irglová had previously performed music as the Swell Season, and composed and performed the film's original songs.\"]], ['The Cat and the Fiddle (film)', ['The Cat and the Fiddle is a 1934 American Pre-Code romantic musical film directed by William K. Howard based on the hit 1931 Broadway musical of the same name by Jerome Kern and Otto A. Harbach, about a romance between a struggling composer and an American singer.', ' The film stars Ramon Novarro and Jeanette MacDonald in her MGM debut.']], ['Ratha Sapthami', ['Ratha Sapthami (ರಥಸಪ್ತಮಿ) is a 1986 Indian-Kannada romantic musical film directed by M. S. Rajashekar and produced by S. A. Govindaraj.', ' The film starred Shivarajkumar in his second venture after \"Anand\" and debutant actress Asha Rani and playwrighter Parvathavani in the lead roles.', ' The film had a musical score composed by Upendra Kumar while the lyrics, screenplay and dialogues were written by Chi.', ' Udaya Shankar.', ' P. Vasu was co-writer of this movie.', ' The movie is based on Kannada novel of same name by Vidyullatha Sasanoor.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.741\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7fd1515542992e7d278db1', 'answer': 'Province of Canterbury', 'question': 'Grimsby Minster is located in which English Province?', 'supporting_facts': [['Grimsby Minster', 1], ['Diocese of Lincoln', 0]], 'context': [['Religious life at Stonyhurst College', ['Stonyhurst College is Roman Catholic and has had a significant place in English Catholic history for many centuries (including more chequered moments such as the Popish Plot and Gunpowder Plot conspiracies).', ' In 1803 the Society of Jesus was re-established in Britain at Stonyhurst and the school became the headquarters of the English Province.', ' Until the 1920s Jesuit priests were trained on site in what is today the preparatory school.', ' The school continues to place Catholicism and Jesuit philosophy at its core.', ' The present chaplain is Father John Twist, SJ.']], ['Daniel Coxe, Jr.', [\"Colonel Daniel Coxe, Jr. (1673–1739), son of Dr. Daniel Coxe, went to his father's North American lands.\", ' He lived in the American colonies from 1702 to 1716 and after returning to England published an account in 1722 of his travels and a description of the area encompassed by his father\\'s claim, entitled \"A Description of the English Province of Carolana, by the Spaniards called Florida, And by the French La Louisiane\".']], ['Cambridge Blackfriars', ['Cambridge Blackfriars is a priory in Cambridgeshire, England.', ' It was established in 1238, dissolved in 1538 and re-established in 1938.', ' Today it continues to operate as a Dominican priory and the novitiate house of the English Province of the Order of Preachers.']], ['York Shire (Province of New York)', ['The Shire of York (Yorkshire), was the first large governmental unit organized in the English Province of New York soon after English control of the area was established in 1664.']], ['Grimsby Minster', ['Grimsby Minster is a minster and parish church located in Grimsby, North East Lincolnshire, England.', ' Dedicated to St James, the church belongs to the Church of England and is within the Diocese of Lincoln.']], ['Siege of Fort Nashwaak (1696)', [\"The Siege of Fort Nashwaak occurred during King William's War when New England forces from Boston attacked the capital of Acadia, Fort Nashwaak, at present-day Fredericton, New Brunswick.\", ' The siege was in retaliation for the French and Indian Siege of Pemaquid (1696) at present day Bristol, Maine.', ' In the English Province of Massachusetts Bay.', ' Colonel John Hathorne and Major Benjamin Church were the leaders of the New England force of 400 men.', ' The siege lasted two days, between October 18–20, 1696, and formed part of a larger expedition by Church against a number of other Acadian communities.']], [\"St James' School, Grimsby\", [\"St James' School is a coeducational independent day and boarding school located in Grimsby, North East Lincolnshire, England.\", ' It comprises a Preparatory School, Senior School and Sixth Form.', \" The school is associated with the Church of England Grimsby Minster, dedicated to St James, and incorporates education for the Minster's choristers.\"]], ['Diocese of Lincoln', ['The Diocese of Lincoln forms part of the Province of Canterbury in England.', ' The present diocese covers the ceremonial county of Lincolnshire.']], ['Timothy Radcliffe', ['Timothy Radcliffe, OP (born 1 August 1947 in London) is a Roman Catholic priest and Dominican friar of the English Province, and former Master of the Order of Preachers from 1992 to 2001.', \" He is the only member of the English Province of the Dominicans to have held the office since the Order's foundation in 1216.\", ' He is currently the Director of the Las Casas Institute of Blackfriars, Oxford which focuses on the promotion of Social Justice and Human Rights.']], ['Raid on Chignecto (1696)', [\"The Raid on Chignecto occurred during King William's War when New England forces from Boston attacked the Isthmus of Chignecto, Acadia in present-day Nova Scotia.\", ' The raid was in retaliation for the French and Indian Siege of Pemaquid (1696) at present day Bristol, Maine.', ' In the English Province of Massachusetts Bay.', ' Colonel Benjamin Church was the leader of the New England force of 400 men.', ' The raid lasted nine days, between September 20–29, 1696, and formed part of a larger expedition by Church against a number of other Acadian communities.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.742\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae713af554299572ea546bb', 'answer': 'Soviet Union', 'question': 'Where did Mark Pavelich win Gold in 1980?', 'supporting_facts': [['Mark Pavelich', 0], ['Miracle on Ice', 0]], 'context': [['Indonesia at the 1992 Summer Olympics', ['Indonesia competed at the 1992 Summer Olympics in Barcelona, Spain.', ' 42 competitors, 27 men and 15 women, took part in 31 events in 10 sports.', \" They won both Men's and Women's singles as their first gold medal here from badminton that appeared for the first time at the summer Olympics.\", ' They made it the first time Indonesia won gold at the Olympics, and also made Indonesia the first Southeast Asian country to win gold.', ' Alan Budikusuma and Susi Susanti, who later married, had the Olympic anthem played on their wedding day.']], ['Liliyana Natsir', ['Liliyana Natsir (born 9 September 1985) is an Indonesian female badminton player who specializes in doubles.', ' With one gold and silver from Olympic Games and four gold medals at BWF World Championships, she is regarded as one of the greatest mixed doubles players in the history.', ' She gained huge success by partnering with two different players.', ' Her current partner is Tontowi Ahmad since 2011, after separating from her previous partner, Nova Widianto.', ' She also won three titles in a row from 2012–2014 at All England Badminton Championships, one of the most prestigious and oldest tournament in the sport.', ' Been entering the top level since 18 years old, her tactical awareness, game vision, and dominance at the front court are considered as one of the best in the tour.', ' In 2016, she and Tontowi Ahmad became the first Indonesian mixed doubles pair to win gold medal at the Olympic Games by beating Chan Peng Soon and Goh Liu Ying of Malaysia.']], ['Lauren Mitchell', ['Lauren Mitchell (born 23 July 1991) is a retired Australian artistic gymnast.', ' She is the 2010 World Champion on floor and the 2009 World Championships silver medalist on balance beam and floor.', ' Mitchell is only the second Australian woman gymnast to win medals at a World Championships, and the first to win gold.']], ['Caroline Ouellette', ['Caroline Ouellette (born May 25, 1979) is a Canadian ice hockey player.', \" She is a member of the Canadian national women's ice hockey team and a member of Canadiennes de Montreal (CWHL).\", ' 4-time Olympic Gold Medallist / 5-time World Championships Gold / 4-time World Championships Silver / 2-time Clarkson Cup Champion, Ouellette is currently in the Top 10 in all-time NCAA scoring with 229 career points.', \" Ouellette is a member of the Triple Gold Club (the accomplishment is not yet officially recognized by the IIHF for women), as one of only three women to win the Clarkson Cup, a gold medal at the Olympic Winter Games, and gold at the IIHF women's world hockey championships.\", ' She is also one of only five athletes to win gold in four consecutive Winter Games, along with teammates Jayna Hefford and Hayley Wickenheiser.']], ['Jeon Hyeok-jin', ['Jeon Hyeok-jin (born 13 June 1995) is a Korean badminton player.', ' Specializing in singles, he was runner-up at the 2013 Asian Junior Badminton Championships.', \" He was part of the Korean team that won men's team gold at the 2014 Asian Games and the following year, he beat both Chou Tien-chen and compatriot Son Wan-ho to win gold at the 2015 Summer Universiade.\", ' In 2017, he helped the Korean national team to win the world team championships at the Sudirman Cup.']], ['Kim Dotson', ['Kim Dotson is a former national level competitor in taekwondo from Cleveland, Ohio, United States.', ' Kim would win the 1985 World Cup.', ' Kim would win gold at the 1986 PanAmerican Championships.', ' Kim would compete in the 1988 Seoul Olympic Games.', ' and would win Silver at both the 1987 and 1989 World Taekwondo Championships.', ' Kim would serve as a coach for several women in taekwondo.']], ['Valerie Brisco-Hooks', ['Valerie Ann Brisco-Hooks (born July 6, 1960 in Greenwood, Mississippi) won three gold medals as an Olympic track and field athlete at the 1984 Olympics at Los Angeles, California, making her the first Olympian to win gold medals in both the 200- and 400-meter races at a single Olympics.']], ['Sam Stoller', ['Sam Stoller (August 8, 1915 – May 29, 1985) was an American sprinter and long jumper who tied the world record in the 60-yard dash in 1936.', ' He is best known for his exclusion from the American 4\\xa0×\\xa0100 relay team at the 1936 Summer Olympics in Berlin, triggering widespread speculation that he and Marty Glickman, the only two Jews on the U.S. track team, were excluded because U.S. Olympic Committee chairman Avery Brundage wanted to avoid embarrassing Adolf Hitler by having two Jewish athletes win gold medals.', ' Stoller vowed at the time that he would never run again, but he returned in 1937 to win both the Big Ten Conference and NCAA championships in the 100-yard dash.', ' After graduating from the University of Michigan in 1937, Stoller briefly went into a singing and acting career as \"Singin\\' Sammy Stoller.\"']], ['Mark Pavelich', ['Mark Thomas Pavelich (born February 28, 1958 in Eveleth, Minnesota) is a retired US professional ice hockey forward who played 355 regular season games in the NHL for the New York Rangers, Minnesota North Stars and San Jose Sharks between 1981 and 1992 and was a member of the 1980 U.S. Olympic hockey team that won the gold medal in what has been called the \"Miracle on Ice\".']], ['Miracle on Ice', ['The \"Miracle on Ice\" refers to a medal-round game during the men\\'s ice hockey tournament at the 1980 Winter Olympics in Lake Placid, New York, played between the hosting United States, and the defending gold medalists, the Soviet Union.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.742\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5addc7e35542997545bbbdbe', 'answer': 'American Samoa, but not on all Native American tribal lands', 'question': 'Which areas of the United States were still able to deny sames sex marriages after the case in which Edith \"Edie\" Windsor was the main plaintiff?', 'supporting_facts': [['Edith Windsor', 1], ['Same-sex marriage in the United States', 0]], 'context': [['Same-sex marriage in the United States', ['In the United States, same-sex marriage is legal in all states, Washington, D.C., as well as all U.S. territories except American Samoa, but not on all Native American tribal lands, since June 26, 2015, when the United States Supreme Court ruled in \"Obergefell v. Hodges\" that state-level bans on same-sex marriage are unconstitutional.', ' The court ruled that the denial of marriage licenses to same-sex couples and the refusal to recognize those marriages performed in other jurisdictions violates the Due Process and the Equal Protection clauses of the Fourteenth Amendment of the United States Constitution.', ' The ruling overturned a 1972 precedent, \"Baker v. Nelson\".', \" Just prior to the Supreme Court's ruling in 2015, same-sex marriage was legal in many but not all U.S. jurisdictions.\"]], ['Gin Chow', ['Gin Chow (1857 - June 1933) was a Chinese immigrant who gained fame in California as a prophet and fortune teller able to predict the weather and other natural events.', ' Chow is credited with successfully predicting the 1925 Santa Barbara earthquake.', ' Chow was also the main plaintiff in the California Supreme Court case \"Gin Chow v. City of Santa Barbara\" which still ranks as one of the most important water rights cases in the state.']], ['Grant Commercial Historic District (Grant, Iowa)', ['The Grant Commercial Historic District is a nationally recognized historic district located in Grant, Iowa, United States.', ' It was listed on the National Register of Historic Places in 2002.', ' At the time of its nomination it contained 17 resources, which included 15 contributing buildings, two contributing structures, and one non-contributing building.', \" The historic district covers the town's central business district.\", ' Grant is a small town located in northeast Montgomery County in the southwest quadrant of the state.', ' It was plated in 1858, and it was known as Milford until the early 20th century even though its post office was Grant.', ' While not on a railroad, the town was still able to maintain a viable commercial district.']], ['Human trafficking in Taiwan', ['Taiwan is primarily a destination for men, women, and children trafficked for the purposes of forced labor and sexual exploitation.', ' It is also a source of women trafficked to Japan, Australia, the United Kingdom, and the United States.', ' Women and girls from the People’s Republic of China (P.R.C.) and Southeast Asian countries are trafficked to Taiwan through fraudulent marriages, deceptive employment offers, and illegal smuggling for sexual exploitation and forced labor.', ' Many trafficking victims are workers from rural areas of Vietnam, Thailand, Indonesia, and the Philippines, employed through recruitment agencies and brokers to perform low skilled work in Taiwan’s construction, fishing, and manufacturing industries, or to work as domestic servants.', ' Such workers are often charged high job placement and service fees, up to $14,000, resulting in substantial debt that labor brokers or employers use as a tool for involuntary servitude.', ' Many foreign workers remain vulnerable to trafficking because legal protections, oversight by authorities and enforcement efforts are inadequate.', ' Taiwan authorities reported that traffickers continued to use fraudulent marriages to facilitate labor and sex trafficking, despite increased efforts by the authorities to prevent this practice.', ' Some women who are smuggled onto Taiwan to seek illegal work were sometimes sold in auctions to sex traffickers, and subsequently forced to work in the commercial sex industry.', ' NGOs reported a sharp increase during the reporting period in the number of boys rescued from prostitution, mainly discovered during police investigations of online social networking sites suspected of being front operations for prostitution rings.']], ['Sea turtle migration', ['Sea turtle migration refers to the long-distance movements of sea turtles (superfamily Chelonioidea) notably as adults but may also refer to the offshore migration of hatchings.', ' Sea turtle hatchings emerge from underground nests and crawl across the beach towards the sea.', ' They then maintain an offshore heading until they reach the open sea.', ' The feeding and nesting sites of adult sea turtles are often distantly separated meaning some must migrate hundreds or even thousands of kilometres.', ' Several main patterns of adult migration have been identified.', ' Some such as the green sea turtle shuttle between nesting sites and coastal foraging areas.', ' The loggerhead sea turtle uses a series of foraging sites.', ' Others such as the leatherback sea turtle and olive ridley sea turtle do not show fidelity to any specific coastal foraging site.', ' Instead, they forage in the open sea in complex movements apparently not towards any goal.', ' Although the foraging movements of leatherbacks seem to be determined to a large part by passive drift with the currents, they are still able to return to specific sites to breed.', ' The ability of adult sea turtles to travel to precise targets has led many to wonder about the navigational mechanisms used.', \" Some have suggested that juvenile and adult turtles might use the Earth's magnetic field to determine their position.\", ' There is evidence for this ability in juvenile green sea turtles.']], ['Market share liability', ['Market share liability is a legal doctrine that allows a plaintiff to establish a prima facie case against a group of product manufacturers for an injury caused by a product, even when the plaintiff does not know from which defendant the product originated.', \" The doctrine is unique to the law of the United States and apportions liability among the manufacturers according to their share of the market for the product giving rise to the plaintiff's injury.\"]], ['Capron v. Van Noorden', ['Capron v. Van Noorden, 6 U.S. 126 (1804) , was a United States Supreme Court case in which the Court allowed a plaintiff to dismiss a case that he had lost at trial because of a lack of diversity jurisdiction, leaving the plaintiff free to bring the case again.']], ['Barnes v. Yahoo!, Inc.', ['Barnes v. Yahoo!, Inc., 570 F. 3d 1096 (D. Or.', ' Nov. 8, 2005), is a United States Court of Appeals for the Ninth Circuit case in which the Ninth Circuit held that Section 230 of the Communications Decency Act (CDA) rules that Yahoo!, Inc., as an Internet service provider cannot be held responsible for failure to remove objectionable content posted to their website by a third party.', \" Plaintiff Cecilia Barnes made claims arising out of Defendant Yahoo!, Inc.'s alleged failure to honor promises to remove offensive content about the plaintiff posted by a third party.\", ' The content consisted of a personal profile with nude photos of the Plaintiff and her contact information.', \" The United States District Court for the District of Oregon had dismissed Barnes' complaint.\"]], ['Lujan v. G &amp; G Fire Sprinklers, Inc.', ['Lujan v. G & G Fire Sprinklers, Inc., 532 U.S. 189 (2001), was a United States Supreme Court case decided in 2001.', ' The case concerned a provision of the California Labor Code which allowed the state to withhold payment to contractors or subcontracters if found in breach of contract, without a specific hearing on the matter.', ' The Court upheld the provision because the companies were still able to pursue a claim in state court.']], ['Edith Windsor', ['Edith \"Edie\" Windsor (née Schlain; June 20, 1929 – September 12, 2017) was an American LGBT rights activist and a technology manager at IBM.', ' She was the lead plaintiff in the Supreme Court of the United States case \"United States v. Windsor\", which successfully overturned Section 3 of the Defense of Marriage Act and was considered a landmark legal victory for the same-sex marriage movement in the United States.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.743\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a72e28f5542992359bc31ba', 'answer': 'outlined by Joel Greenblatt', 'question': 'Which technique did the director at Pzena Investment Management outline?', 'supporting_facts': [['Magic formula investing', 0], ['Joel Greenblatt', 3]], 'context': [['Joel Greenblatt', ['Joel Greenblatt (born December 13, 1957) is an American academic, hedge fund manager, investor, and writer.', ' He is a value investor, and adjunct professor at the Columbia University Graduate School of Business.', ' He is the former chairman of the board of Alliant Techsystems and founder of the New York Securities Auction Corporation.', ' He is also a director at Pzena Investment Management, a high-end value firm.']], ['Orbis Investment Management', ['Orbis Investment Management is an investment management firm headquartered in Bermuda, with offices in London, Vancouver, Sydney, San Francisco, Hong Kong, Tokyo and Luxembourg.', ' The company has a close relationship with Allan Gray Investment Management in South Africa and Allan Gray Australia.', ' Orbis manages approximately $25\\xa0billion on behalf of both institutional and individual investors.', ' Orbis Access, its direct-to-consumer platform, was launched in the UK in January 2015.']], ['Richard Pzena', ['Richard \"Rich\" Pzena (born January 8, 1959) is an American investment manager.', ' He is the founder and chief investment officer of Pzena Investment Management, a New York-based deep value investment firm with $26.4 billion in assets under management.']], ['Journal of Investment Management', ['The Journal of Investment Management (JOIM) is a quarterly refereed journal which seeks to be a nexus of theory and practice of investment management.', ' \"The Journal Of Investment Management\" offers in-depth research with practical significance utilising concepts from the economics and accounting disciplines.', ' The editor is Gifford H. Fong, founder of Gifford Fong Associates, a boutique bond and equity analysis firm.']], ['Separately managed account', ['A separately managed account (SMA) is a term within the investment management industry encompassing several different types of investment accounts.', ' For example, an SMA often is used to refer to an individual managed investment account often offered by a brokerage firm through one of their brokers or financial consultants and managed by independent investment management firms (often called money managers for short) and have varying fee structures.', ' These particular types of SMAs may be called \"wrap fee\" or \"dual contract\" accounts, depending on their structure.', ' There is no official designation for the SMA, but there are common characteristics that are represented in many types of SMA programs.', \" These characteristics include an open structure or flexible investment security choices; multiple money managers; and a customized investment portfolio formulated for a client's specific investment objectives or desired restrictions.\"]], ['Magic formula investing', ['Magic formula investing is a term referring to an investment technique outlined by Joel Greenblatt that uses the principles of value investing.']], ['Royal London Asset Management', ['Royal London Asset Management (RLAM) is a UK-based investment management company with assets under management of more than £101 billion.', ' Headquartered in London, United Kingdom, it has over 2,900 employees working across seven sites in UK and Ireland(as at 30 September 2016).', ' RLAM offers investment management – mutual funds, active and passive portfolio management as well asset allocation for a wide range of clients.', ' RLAM’s clients include, but are not limited to; listed companies, pension schemes, local authorities, educational establishments, charities, wealth managers, financial advisers and multi-managers.', ' RLAM invests across all major asset classes, including the UK and overseas equities, government bonds, investment grade and high yield corporate bonds, property and cash.', \" RLAM is a wholly owned, autonomous subsidiary of the Royal London Group, the UK's largest mutual insurance company.\"]], ['Cowen Group', ['Cowen Inc. is a diversified financial services firm that provides alternative investment management, investment banking, research, and sales and trading services through its two business segments: Cowen Investment Management (formerly Ramius LLC), a global alternative investment management business, and Cowen and Company, LLC, a broker-dealer business.', ' Founded in 1918 by Harry Cowen and Arthur Cowen, Jr., the Firm is headquartered in New York City and has offices located worldwide.']], ['Investment control', ['Investment control or investment controlling is a monitoring function within the asset management, portfolio management or investment management.', ' It is concerned with independently supervising and monitoring the quality of asset management accounts with the aim of ensuring performance and quality in order to provide the required benefit for the asset management client.', ' Dependent on setup, investment controlling not only encompasses controlling activities but also can include areas from compliance to performance review.', ' Investment controlling aspects can also be taken into consideration by asset management clients or investment advisers/consultants and consequently it is likely that these stakeholders also run certain investment controlling activities.']], ['Barclays Wealth', ['Barclays Wealth and Investment Management is a wealth manager providing private banking, investment management, brokerage and fiduciary services to private clients and financial intermediaries all over the world.', ' Barclays provides Wealth and Investment Management across 20 offices to clients in 50 countries and has client assets of £202.8\\xa0billion (as of 30 June 2013).']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.744\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5add1d435542990d50227de8', 'answer': 'Vernier, Switzerland', 'question': 'Where is the company, which designed the Glomar Challenger, based?', 'supporting_facts': [['Glomar Challenger', 1], ['Transocean', 0]], 'context': [['Bristol Racing Biplane', ['The Bristol Racing Biplane was a British single-seat biplane designed to combine the performance of a monoplane but using the strength of the biplane.', ' It was designed by Robert Grandseigne and Léon Versepuy, who were supervised by George Challenger for the British & Colonial Aeroplane Company of Bristol, it crashed on its first flight.']], ['Arctic Challenger', ['Arctic Challenger is a barge which has been converted by Superior Energy Services for use in the Arctic drilling operations of Shell Oil Company.', ' This barge is designed to function as a \"novel engineering solution\" which they refer to as an Arctic Containment System to respond should a blowout event occur at drilling sites in the Beaufort or Chukchi Seas.', ' According to testimony provided to Senator Mark Begich on 11 October 2012, Coast Guard Rear Admiral Thomas Ostebo said the certification for the Shell spill barge Arctic Challenger to operate in Alaska was given on the 10th of October at the Bellingham, Washington shipyard where it was constructed.', ' Ostebo is commander of the Coast Guard’s 17th district, which covers Alaska.']], ['Containment dome', ['A containment dome is a component of the system designed to contain the underwater blowout of an oil well such as occurred with the Macondo Well blowout from the Deepwater Horizon oil spill.', ' This portion of the system is designed as a vacuum to suck up the products being expelled from a blowout and deliver those products to the containment system housed on the vessel moored above the blowout.', ' Superior Energy Services is constructing this device to be used by Shell Oil Company on the barge \"Arctic Challenger\" as their \"fourth line of defense\" against a blowout in the Arctic drilling regions in the Chukchi Sea and Beaufort Sea.']], ['Deepsea Challenger', ['Deepsea Challenger (DCV 1) is a 7.3 m deep-diving submersible designed to reach the bottom of Challenger Deep, the deepest-known point on Earth.', ' On 26 March 2012, Canadian film director James Cameron piloted the craft to accomplish this goal in the second manned dive reaching the Challenger Deep.', ' Built in Sydney, Australia by the research and design company Acheron Project Pty Ltd, \"Deepsea Challenger\" includes scientific sampling equipment and high-definition 3-D cameras, and reached the ocean\\'s deepest point after two hours and 36 minutes of descent from the surface.']], ['JOIDES Resolution', ['JOIDES Resolution (Joint Oceanographic Institutions for Deep Earth Sampling), often abbreviated JR, is one of two scientific drilling ships used by the International Ocean Discovery Program (the other being the Japanese drilling vessel Chikyu).', ' The JR was previously the main research ship used in the Ocean Drilling Program and was used along with the Chikyu throughout the Integrated Ocean Drilling Program.', ' She is the successor of \"Glomar Challenger\".']], ['Transocean', [\"Transocean Ltd. is one of the world's largest offshore drilling contractors and is based in Vernier, Switzerland.\", ' The company has offices in 20 countries, including Switzerland, Canada, United States, Norway, Scotland, India, Brazil, Singapore, Indonesia and Malaysia.']], ['Glomar Challenger', ['Glomar Challenger was a deep sea research and scientific drilling vessel for oceanography and marine geology studies.', ' The drillship was designed by Global Marine Inc. (now Transocean Inc.) specifically for a long term contract with the American National Science Foundation and University of California Scripps Institution of Oceanography and built by Levingston Shipbuilding Company in Orange, Texas.', ' Launched on March 23, 1968, the vessel was owned and operated by the Global Marine Inc. corporation.', ' \"Glomar Challenger\" was given its name as a tribute to the accomplishments of the oceanographic survey vessel HMS \"Challenger\" .', ' Glomar is a truncation of Global Marine.']], ['Track Marshall', ['Track Marshall was a brand of earthmoving equipment who were active during WWII building tanks. Later, they produced a range of crawler tractors, based on the wheeled tractor \"Field Marshall\" brand.', ' In 1956 the first Track Marshall bulldozer model was introduced and later the Challenger 3 followed by the Challenger 33, the \"TM55\" and also the successful six-cylinder \"TM70\" (probably equivalent to a Caterpillar D5).', ' They also built the TM 955 track loader (a \"drott\") and also a range of rubber tracked bulldozers.', ' The company was in business for some 50 years, eventually closing in 1990 after new machines replaced older technology.']], ['CMC Zinger', ['The CMC Zinger (, originally the Mitsubishi Zinger before 2015) is a compact MPV designed by Mitsubishi Motors in conjunction with the China Motor Corporation from Taiwan, based on the chassis of the Mitsubishi Challenger, and sold in Taiwan from December 24, 2005.', ' The name derives from a \"person or something full of energy and vitality\".', ' Since June 2007 it has also been marketed in the Philippines as the Mitsubishi Fuzion, as the company claims it \"merges together the best characteristics of [three] vehicles, the sporty character and ruggedness of an SUV, the spaciousness and versatility of a van, and riding comfort of a passenger car\".']], ['Glomar Challenger Basin', ['Glomar Challenger Basin ( ) is a northeast trending undersea basin in the central Ross Sea continental shelf named for the research ship \"Glomar Challenger\".', ' The name was approved by the Advisory Committee for Undersea Features in June 1988.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.745\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8f55f9554299458435d5bd', 'answer': 'actor', 'question': 'What profession did Willi Forst and Elmer Clifton share?', 'supporting_facts': [['Willi Forst', 0], ['Elmer Clifton', 0]], 'context': [['Kaiserjäger (film)', ['Kaiserjäger is a 1956 Austrian film directed by Willi Forst.']], ['Gently My Songs Entreat', ['Gently My Songs Entreat (German: Leise flehen meine Lieder ) is a 1933 Austrian-German musical film directed by Willi Forst and starring Marta Eggerth, Luise Ullrich and Hans Jaray.', ' Art direction was by Julius von Borsody.', ' The film is a biopic of the composer Franz Schubert (1797–1828).', \" It was Forst's directorial debut.\", ' A British version was made called \"Unfinished Symphony\".', ' The German title refers to the first line of the Lied \"Ständchen\" (Serenade) from Schubert\\'s collection \"Schwanengesang\", \"the most famous serenade in the world\", which Eggerth performs in the film.']], ['Operetta (film)', ['Operetta (German: Operette) is a 1940 musical film directed by Willi Forst and starring Forst, Maria Holst and Dora Komar.', ' The film was made by Wien-Film, a Vienna-based company set up after Austria had been incorporated into Greater Germany following the 1938 Anschluss.', ' It is the first film in director Willi Forst\\'s \"Viennese Trilogy\" followed by \"Vienna Blood\" (1942) and \"Viennese Girls\" (1945).', ' The film portrays the life of Franz Jauner (1832–1900), a leading musical figure in the city.', ' It is both an operetta film and a Wiener Film.']], ['Elmer Clifton', ['Elmer Clifton (March 14, 1890 – October 15, 1949) was an American writer, director and actor from the early silent days.', ' A collaborator of D.W. Griffith, he appeared in \"The Birth of a Nation\" (1915) and \"Intolerance\" (1916) before giving up acting in 1917 to concentrate on work behind the camera, with Griffith and Joseph Henabery as his mentors.', ' His first feature-length solo effort as a director was \"The Flame of Youth\" with Jack Mulhall.']], ['Miracles Still Happen (1951 film)', ['Miracles Still Happen (German: Es geschehen noch Wunder) is a 1951 West German romantic comedy film directed by Willi Forst and starring Forst, Hildegard Knef and Marianne Wischmann.', ' It was intended by Forst as a more harmless follow-up to his controversial \"Die Sünderin\" which had also starred Knef.']], ['The Prince of Arcadia', ['The Prince of Arcadia (German: Der Prinz von Arkadien) is a 1932 Austrian-German romance film directed by Karl Hartl and starring Willi Forst, Liane Haid and Hedwig Bleibtreu.', ' It premiered on 18 May 1932.']], ['Burgtheater (film)', ['Burgtheater is a 1936 Austrian drama film directed by Willi Forst.', ' Most of the film was shot in the Burgtheater in Vienna.']], ['Viennese Girls', ['Viennese Girls (German:Wiener Mädeln) is a 1945 historical musical film directed by Willi Forst and starring Forst, Anton Edthofer and Judith Holzmeister.', ' The film was made by Wien-Film, a Vienna-based company set up after Austria had been incorporated into Greater Germany following the 1938 Anschluss.', ' It was the third film in Forst\\'s \"Viennese Trilogy\" which also included \"Operetta\" (1940) and \"Vienna Blood\" (1942).', ' The film was finished in 1945, during the closing days of the Second World War.', ' This led to severe delays in its release, which eventually took place in 1949 in two separate versions.', ' One was released by the Soviet-backed Sovexport in the Eastern Bloc and the other by Forst.']], [\"A Student's Song of Heidelberg\", [\"A Student's Song of Heidelberg (German:Ein Burschenlied aus Heidelberg) is a 1930 German musical film directed by Karl Hartl and starring Hans Brausewetter, Betty Bird and Willi Forst.\", \" It marked Hartl's directoral debut.\", ' The film is in the tradition of the nostalgic Old Heidelberg.']], ['Willi Forst', ['Willi Forst, born Wilhelm Anton Frohs (7 April 1903 – 11 August 1980) was an Austrian actor, screenwriter, film director, film producer and singer.', ' As a debonair actor he was a darling of the German-speaking film audiences, as a director, one of the most significant makers of the Viennese period musical melodramas and comedies of the 1930s known as \"Wiener Filme\".', ' From the mid-1930s he also recorded many records, largely of sentimental Viennese songs, for the Odeon Records label owned by Carl Lindström AG.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.745\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7322a25542991f9a20c634', 'answer': 'The Metropolitan Life Insurance Company Tower', 'question': 'Was the Metropolitan Life Insurance Company Tower [Met Life Tower] or the 15 Hudson Yards building designed by the firm of Napoleon LeBrun & Sons?', 'supporting_facts': [['Metropolitan Life Insurance Company Tower', 0], ['Metropolitan Life Insurance Company Tower', 1], ['15 Hudson Yards', 1]], 'context': [['Supreme Life Building', ['The Supreme Life Building is a historic insurance building located at 3501 S. Dr. Martin Luther King Drive in the Douglas community area of Chicago, Illinois.', ' Built in 1921, the building served as the headquarters of the Supreme Life Insurance Company, which was founded two years earlier.', ' The company, originally known as the Liberty Life Insurance Company, was the first African-American owned insurance company in the northern United States.', \" Since white-owned insurance firms regularly denied black customers life insurance when the firm was founded, the firm played an important role in providing life insurance to Chicago's African-American community.\", \" The company ultimately became the largest African-American owned business in the northern states and became a symbol of the predominantly black Bronzeville neighborhood's economic success from the 1920s to the 1950s.\"]], ['Napoleon LeBrun', [\"Napoleon Eugene Charles Henry LeBrun (January 2, 1821 – July 9, 1901) was an American architect known for several notable Philadelphia churches, in particular St. Augustine's Church on Fourth Street and the Cathedral-Basilica of Sts.\", ' Peter and Paul on Logan Square.', ' He also designed the Academy of Music at Broad and Locust Streets.', ' LeBrun later moved to New York City, where he established the firm Napoleon LeBrun & Sons, which designed numerous notable buildings.']], ['Metropolitan Life Insurance Company Tower', ['The Metropolitan Life Insurance Company Tower, colloquially known as the Met Life Tower, is a landmark skyscraper located on Madison Avenue near the intersection with East 23rd Street, across from Madison Square Park in Manhattan, New York City.', ' Designed by the architectural firm of Napoleon LeBrun & Sons and built by the Hedden Construction Company, the tower is modeled after the Campanile in Venice, Italy.', ' The hotel located in the clock tower portion of the building has the address 5 Madison Avenue, while the office building covering the rest of the block, occupied primarily by Credit Suisse, is referred to as 1 Madison Avenue.']], ['Hedden Construction Company', ['Some of the finest buildings in New Jersey, New York City, and other large eastern cities were built by the Hedden Construction Company, one of the largest construction companies operating in Newark in the very early 1900s.', ' Among the most notable is the Metropolitan Life Insurance Company Tower located at One Madison Avenue in New York, NY.', \" The tower was the world's tallest building from 1909 to 1913 and home to the Hedden Construction Company's main offices located on the 36th and 37th floors.\", ' During this prosperous period over $40,000,000 in construction contracts and payments were collected by the firm.']], ['15 Hudson Yards', [\"15 Hudson Yards is a residential building currently under construction on Manhattan's West Side.\", \" Located in Chelsea near Hell's Kitchen Penn Station area, the building is a part of the Hudson Yards project, a plan to redevelop the Metropolitan Transportation Authority's West Side Yards.\", ' The tower started construction on December 4, 2014.']], ['Flatiron Building', ['The Flatiron Building, originally the Fuller Building, is a triangular 22-story steel-framed landmarked building located at 175 Fifth Avenue in the borough of Manhattan, New York City, and is considered to be a groundbreaking skyscraper.', ' Upon completion in 1902, it was one of the tallest buildings in the city at 20 floors high and one of only two skyscrapers north of 14th Street – the other being the Metropolitan Life Insurance Company Tower, one block east.', \" The building sits on a triangular block formed by Fifth Avenue, Broadway, and East 22nd Street, with 23rd Street grazing the triangle's northern (uptown) peak.\", ' As with numerous other wedge-shaped buildings, the name \"Flatiron\" derives from its resemblance to a cast-iron clothes iron.']], ['Protective Life', ['Protective Life Corporation is a financial service holding company in Birmingham, Alabama.', ' The company’s primary subsidiary, Protective Life Insurance Company, was established in 1907 and now markets its products and services in all 50 states.', ' As of December 31, 2016, the corporation had more than 2,700 employees, annual revenues of $4.48 billion and assets of $75 billion.', \" In addition to Protective Life Insurance Company, Protective Life Corporation's subsidiaries include West Coast Life Insurance Company, MONY Life Insurance Company, Protective Life And Annuity Insurance Company, ProEquities Inc./Protective Securities, and Lyndon Property Insurance Company.\"]], ['Physicians Mutual', ['Physicians Mutual is a privately held insurance company headquartered in Omaha, Nebraska, United States, that consists of Physicians Mutual Insurance Company and Physicians Life Insurance Company.', ' Founded as Physicians Mutual Insurance Company in 1902 by Edwin E. Elliott, Physicians Mutual began by selling health insurance to medical professionals.', ' Policies were offered to the general public starting in 1962, and by 1970 the company expanded into life insurance when it founded Physicians Life Insurance Company.', ' Today the company offers a variety of insurance products, annuities, Medicare, Medigap, Medicare Supplement, Term Life Insurance, Whole Life Insurance, Cancer and funeral pre-planning services.', ' It holds over US$3 billion in assets and employs over one thousand people.', ' Robert A. Reed is chief executive officer and president.']], ['Lyceum Theatre (Park Avenue South)', ['The Lyceum Theatre was a theatre in New York City located on Fourth Avenue, now Park Avenue South, between 23rd and 24th Streets in Manhattan.', ' It was built in 1885 and operated until 1902, when it was torn down to make way for the Metropolitan Life Insurance Company Tower.', ' It was replaced by a new Lyceum Theatre on 45th Street.', ' For most of its existence, the theatre was home to Daniel Frohman’s Lyceum Theatre Stock Company, which presented many important plays and actors of the day.']], ['Metropolitan Life North Building', ['The Metropolitan Life North Building, now known as Eleven Madison, is a 30-story art deco skyscraper on Madison Square Park in Manhattan, New York City, at 11-25 Madison Avenue.', ' The building is bordered by East 24th Street, Madison Avenue, East 25th Street and Park Avenue South, and is connected by an elevated walkway to the Met Life Tower just south of it.', \" The North Building was built on the site of Richard Upjohn's original Madison Square Presbyterian Church.\", ' The second church, designed by Stanford White of McKim, Mead and White was built in 1906, across 24th street on land conveyed by Metropolitan Life.', ' As part of the Metropolitan Life Home Office Complex, the North Building was added to the National Register of Historic Places on January 19, 1996.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.746\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abc23c45542993a06baf896', 'answer': 'Salzkammergut', 'question': \"Which region of Austria means 'Estate of the Salt Chamber' and covers the village of Obertraun, a popular destination for skiing and snowboarding in the winter?\", 'supporting_facts': [['Obertraun', 2], ['Salzkammergut', 3]], 'context': [['Salzkammergut', ['The Salzkammergut is a resort area located in Austria.', ' It stretches from the City of Salzburg eastwards along the Austrian Alpine Foreland and the Northern Limestone Alps to the peaks of the Dachstein Mountains, spanning the federal states of Upper Austria, Salzburg, and Styria.', ' The main river of the region is the Traun, a right tributary of the Danube.', ' The name \"Salzkammergut\" literally means \"Estate of the Salt Chamber\" and derives from the Imperial Salt Chamber, the authority charged with running the precious salt mines of the Habsburg Monarchy.', ' It is a UNESCO World Heritage Site.']], ['Aspen/Snowmass', ['Aspen Snowmass is a winter resort complex located in Pitkin County in western Colorado in the United States.', ' Owned and operated by the Aspen Skiing Company it comprises four skiing/snowboarding areas on four adjacent mountains in the vicinity of the towns of Aspen and Snowmass Village.', ' The four areas collectively form one of the most famous winter resorts in the world and are annually the destination for visitors from all over the world.']], ['List of Olympic medalists in snowboarding', ['Snowboarding is a sport that has been contested at the Winter Olympic Games since the 1998 Winter Olympics in Nagano, Japan.', ' Snowboarding was one of five new sports or disciplines added to the Winter Olympic programme between 1992 and 2002, and was the only one not to have been a previous medal or demonstration event.', ' In 1998, four events, two for men and two for women, were held in two specialities: the giant slalom, a downhill event similar to giant slalom skiing; and the half-pipe, in which competitors perform tricks while going from one side of a semi-circular ditch to the other.', \" Canadian Ross Rebagliati won the men's giant slalom and became the first athlete to win a gold medal in snowboarding.\", ' Rebagliati was briefly stripped of his medal by the International Olympic Committee (IOC) after testing positive for marijuana.', \" However, the IOC's decision was reverted following an appeal from the Canadian Olympic Association.\", ' For the 2002 Winter Olympics, the giant slalom was dropped in favour of the parallel giant slalom, an event that involves head-to-head racing.', ' In 2006, a third event, the snowboard cross, was held for the first time.', ' In this event, competitors race against each other down a course with jumps, beams and other obstacles.']], ['Obertraun', ['Obertraun, Upper Austria is a village in the Salzkammergut, a region in Austria.', ' It is located near the Hallstätter See (Hallstatt Lake) and Hoher Dachstein.', ' Obertraun is a popular holiday destination offering activities such as skiing, snowboarding in the winter and mountain biking, swimming and kayaking in the summer.']], ['Estate jewelry', ['Estate Jewelry (or jewellery) is a term used, most commonly in a retail sense, to refer to jewelry and often timepieces which are part of the ‘estate’ of a deceased person.', ' More correctly estate jewelry is second-hand or pre-owned jewelry, with the ‘estate’ appellation signifying that the item is antique, vintage or an otherwise considered a significant or important piece.']], ['Front Range', ['The Front Range is a mountain range of the Southern Rocky Mountains of North America located in the central portion of the U.S. State of Colorado, and southeastern portion of the U.S. State of Wyoming.', ' It is the first mountain range encountered moving west along the 40th parallel north across the Great Plains of North America.', ' The Front Range runs north-south between Casper, Wyoming and Pueblo, Colorado and rises nearly 10,000 feet above the Great Plains.', ' Longs Peak, Mount Evans, and Pikes Peak are its most prominent peaks, visible from the Interstate 25 corridor.', ' The area is a popular destination for mountain biking, hiking, climbing, and camping during the warmer months and for skiing and snowboarding during winter.', ' Millions of years ago the present-day Front Range was home to ancient mountain ranges, deserts, beaches, and even oceans.']], ['Snowboarding at the Winter Olympics', ['Snowboarding is a sport at the Winter Olympic Games.', ' It was first included in the 1998 Winter Olympics in Nagano, Japan.', ' Snowboarding was one of five new sports or disciplines added to the Winter Olympic program between 1992 and 2002, and was the only one not to have been a previous medal or demonstration event.', ' In 1998, four events, two for men and two for women, were held in two specialities: the giant slalom, a downhill event similar to giant slalom skiing; and the half-pipe, in which competitors perform tricks while going from one side of a semi-circular ditch to the other.', \" Canadian Ross Rebagliati won the men's giant slalom and became the first athlete to win a gold medal in snowboarding.\", ' Rebagliati was briefly stripped of his medal by the International Olympic Committee (IOC) after testing positive for marijuana.', \" However, the IOC's decision was reverted following an appeal from the Canadian Olympic Association.\", ' For the 2002 Winter Olympics, giant slalom was expanded to add head-to-head racing and was renamed parallel giant slalom.', ' In 2006, a third event, the snowboard cross, was held for the first time.', ' In this event, competitors race against each other down a course with jumps, beams and other obstacles.', \" On July 11, 2011, the International Olympic Committee's Executive Board approved the addition of Ski and Snowboard Slopestyle to the Winter Olympics roster of events, effective in 2014.\", \" The decision was announced via press conference from the IOC's meeting in Durban, South Africa.\", ' A fifth event, parallel slalom, was added be in 2014.']], ['Snowkiting', ['Snowkiting or Kite skiing is an outdoor winter sport where people use kite power to glide on snow or ice.', ' The skier uses a kite to give them power over large jumps.', ' The sport is similar to water-based kiteboarding, but with the footwear used in snowboarding or skiing.', 'The principes of using the kite is the same, but in different terrain.', ' In the early days of snowkiting, foil kites were the most common type; nowadays many kiteboarders use inflatable kites.', \" However, since 2013, newly developed racing foil kites seem to dominate speed races and expedition races, like Red Bull Ragnarok (held on the Norwegian Hardangervidda plateau) and the Vake mini-expedition race (held at Norway's most northern Varanger peninsula).\", ' Snowkiting differs from other alpine sports in that it is possible for the snowkiter to travel uphill and downhill with any wind direction.', ' Like kiteboarding, snowkiting can be very hazardous and should be learned and practiced with care.', ' Snowkiting is becoming increasingly popular in places often associated with skiing and snowboarding, such as Russia, Canada, Iceland, France, Switzerland, Austria, Norway, Sweden and the Northern and Central United States.', ' The sport is becoming more diverse as adventurers use kites to travel great distances and sports enthusiasts push the boundaries of freestyle, big air, speed and back country exploration.']], ['Crystal Mountain (British Columbia)', ['Crystal Mountain Resort was a small day-use ski area near West Kelowna, British Columbia, Canada.', ' It has two chairlifts and one surface lift: a GMD Mueller doublechair, a Leitner-Poma triplechair, and a Doppelmayr T-bar.', ' The ski area has 30 designated groomed runs and the resort also has many different tree trails and some glades.', ' Despite the smaller size of the resort, which is considerably smaller than neighbouring resorts such as Big White Ski Resort and Silver Star Mountain Resort, the mountain is of good size and is a popular destination for both experienced and first-time skiers, snowboarders and snowshoers.', ' Despite the lower elevation, the resort receives heavy snowfall each year with many powder days throughout the winter season.', ' It was long known as Last Mountain Ski Resort but has since changed its name to Crystal Mountain Resort in 1992.', ' Crystal Mountain Resort offers lessons for skiing and snowboarding as well as offering rentals for snowshoeing.', ' It has not been in operation since 2014 due to a lift malfunction of the Blue doublechair.']], ['Bromley Mountain', ['Bromley Mountain is located in southern Vermont, United States and is part of the Green Mountains.', ' It is located in Bennington County, seven miles (11\\xa0km) east of Manchester, Vermont and just west of Peru, Vermont.', ' It is a popular destination for skiing and snowboarding.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.747\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a750d315542996c70cfae84', 'answer': 'Ronald Ryan', 'question': 'Whose death dramatized in a stage play helped end the death penalty in Australia?', 'supporting_facts': [['Barry Dickins', 1], ['Ronald Ryan', 3]], 'context': [['National Coalition to Abolish the Death Penalty', ['The National Coalition to Abolish the Death Penalty (NCADP) is a large organization dedicated to the abolition of the death penalty in the United States.', ' Founded in 1976 (the same year the death penalty was reinstated by the Supreme Court of the United States) by Henry Schwarzschild, the NCADP is the only fully staffed nationwide organization in the United States dedicated to the total abolition of the death penalty in the country.', ' It also provides extensive information regarding imminent and past executions, death penalty defendants, numbers of people executed in the U.S., as well as a detailed breakdown of the current death row population, and a list of which U.S. state and federal jurisdictions use the death penalty.']], ['Murder (Abolition of Death Penalty) Act 1965', ['The Murder (Abolition of Death Penalty) Act 1965 is an Act of the Parliament of the United Kingdom.', ' It abolished the death penalty for murder in Great Britain (the death penalty for murder survived in Northern Ireland until 1973).', ' The Act replaced the penalty of death with a mandatory sentence of imprisonment for life.']], ['Yoshihiro Yasuda', ['Yoshihiro Yasuda (安田 好弘 \"Yasuda Yoshihiro\", born December 4, 1947) is a famed and controversial lawyer in Japan who is known for his anti-death penalty activism.', ' With the death penalty being a prominent method of prosecution in the Japanese judicial system for violent criminals, Yasuda has a history of defending many of these criminals as he wishes to prevent the death penalty from being imposed.', ' As an advocate for the abolition of the death penalty, Yasuda has been able to successfully prevent a large number of death sentences from being handed down in his career.', \" At the time Yasuda took on many of these violent cases, such cases were seen as damaging to a lawyer's career, and therefore, there existed only a small number of lawyers who took on such cases because many feared the media bashing, and could not expect much compensation.\", ' A significant number of these cases were then defended by Yasuda, and this concentration was viewed as problematic by some critics.', ' He took part in many of these controversial trials because he believed that the suspects were tried unfairly as a result of the mass media bashings.', ' Yasuda is also known to reject television appearances for he dislikes the mass media.']], ['Troy Leon Gregg', ['Troy Leon Gregg (April 22, 1948 – July 29, 1980) was the first condemned individual whose death sentence was upheld by the United States Supreme Court after the Court\\'s decision in \"Furman v. Georgia\" invalidated all previously enacted death penalty laws in the United States.', ' Gregg was convicted of having murdered Fred Edward Simmons and Bob Durwood Moore in order to rob them.', ' The victims had given him and another man, Dennis Weaver, a ride when they were hitchhiking.', ' The crime occurred on November 21, 1973.']], ['McGautha v. California', ['McGautha v. California, 402 U.S. 183 (1971) is a criminal case heard by the United States Supreme Court, in which the Court held that the lack of legal standards by which juries imposed the death penalty was not an unconstitutional violation of the due process clause portion of the Eighth Amendment.', ' Justice Harlan wrote that writing rules for jury death penalty decisions was beyond current human ability.', ' The context was public and philosophical scrutiny of the unequal application of the death penalty, especially in that black who killed whites were much more likely to have a death penalty imposed.', ' McGautha was overruled one year later by Furman v. Georgia, which held that sentencing discretion must be narrowed \"so as to minimize the risk of wholly arbitrary and capricious action.\"']], ['Ronald Ryan', ['Ronald Joseph Ryan (21 February 1925 – 3 February 1967) was the last person to be legally executed in Australia.', ' Ryan was found guilty of shooting and killing warder George Hodson during an escape from Pentridge Prison, Victoria, in 1965.', \" Ryan's hanging was met with some of the largest public protests in the history of Australia and led to the end of capital punishment.\", ' The death penalty was abolished in 1985.']], ['Barry Dickins', ['Barry Dickins (born 1949) is a prolific Australian playwright, author, artist, actor, educator and journalist, probably best known for his historical dramas and his reminisces about growing up and living in working class Melbourne.', ' His most well-known work is the award winning stage play \"Remember Ronald Ryan\", a dramatization of the life and subsequent death of Ronald Ryan, the last man executed in Australia.', ' He has also written dramas and comedies about other controversial figures like poet Sylvia Plath, opera singer Joan Sutherland, criminal Squizzy Taylor, actor Frank Thring, playwright Oscar Wilde and artist Brett Whiteley.']], ['Campaign to End the Death Penalty', ['The Campaign to End the Death Penalty (CEDP) is an anti-death penalty organization in the United States, built on the philosophy that death row inmates and their family members must be at the center of fighting to abolish the death penalty.', ' According to CEDP, \"Abolition will not come from the desks of local politicians or the power brokers in Washington, whose lives have likely never been touched by the death penalty and whose careers have often been bolstered by it.', ' Abolition can only come from organizing within communities and from people demanding a change.\"']], ['Catholic Church and capital punishment', [\"The Catholic Church's position on capital punishment has varied throughout the centuries following the Church's establishment, evolving from somewhat supportive to largely apathetic to mostly anti-capital punishment.\", ' In more recent times, the Catholic Church has generally moved away from any explicit condoning or approval of capital punishment and has instead increasingly adopted a more disapproving stance on the issue.', ' Modern Church figures such as Pope John Paul II, Pope Francis, and the United States Conference of Catholic Bishops have in fact actively discouraged the death penalty or advocated for the out-right abolition of the death penalty.', ' Historically, the teaching of the Catholic Church used to categorize capital punishment as a form of \"lawful slaying\", a view defended by theological authorities such as Augustine and Thomas Aquinas.', ' Augustine felt that the death penalty was a means of deterring the wicked and protecting the innocent.', ' In the Middle Ages, Thomas Aquinas reaffirmed this position.', ' (See also Aquinas on the death penalty).', ' However, after the Second Vatican Council the Catholic Church has been staunchly opposed to the death penalty.']], ['Capital punishment in Australia', ['Capital punishment in Australia has been abolished in all jurisdictions.', ' Queensland abolished the death penalty in 1922.', ' Tasmania did the same in 1968, the federal government abolished the death penalty in 1973, with application also in the Australian Capital Territory and the Northern Territory.', ' Victoria did so in 1975, South Australia in 1976, and Western Australia in 1984.', ' New South Wales abolished the death penalty for murder in 1955, and for all crimes in 1985.', ' In 2010, the federal government passed legislation prohibiting the re-establishment of capital punishment by any state or territory.', ' Neither the Commonwealth nor any of the states will extradite or deport a prisoner to another jurisdiction if they will face the death penalty, and police co-operation with other countries which have the death penalty has been questioned.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.747\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae0c9dd5542993d6555ec69', 'answer': 'yes', 'question': 'Are Rob Parissi and Robert Pollard both musicians', 'supporting_facts': [['Rob Parissi', 0], ['Robert Pollard', 0]], 'context': [['Play That Funky Music', ['\"Play That Funky Music\" is a song written by Rob Parissi and recorded by the band Wild Cherry.', ' The single was the first release by the Cleveland-based Sweet City record label in April 1976, and distributed by Epic Records.', \" The performers on the recording included lead singer Parissi, electric guitarist Bryan Bassett, bassist Allen Wentz and drummer Ron Beitle, with session players Chuck Berginc, Jack Brndiar (trumpets), and Joe Eckert and Rick Singer (saxes) on the horn riff that runs throughout the song's verses.\", ' The single hit number one on the \"Billboard\" Hot 100 on September 18, 1976, and was also number one on the Hot Soul Singles chart.', ' The single was certified platinum by the Recording Industry Association of America for shipments of over 2 million records, eventually selling 2.5 million in the United States alone.']], ['Get Down Tonight: The Disco Explosion', ['Get Down Tonight: The Disco Explosion was a 2004 musical documentary special which aired on PBS.', ' The special featured Irene Cara, KC & The Sunshine Band, Yvonne Elliman, The Hues Corporation, Peaches & Herb, Karen Lynn Gorney, A Taste of Honey, Rob Parissi of Wild Cherry, Leo Sayer, Deney Terrio, Frankie Valli, Martha Wash, Barry Williams, Norma Jean Wright and Felton Pilate.', ' It was directed by T.J. Lubinsky, and produced by Jerry Blavat, Henry J. DeLuca, Cousin Brucie Morrow and Lubinsky.', ' One of the associate producers was Marty Angelo.']], ['Robert Pollard', ['Robert Ellsworth Pollard Jr. (born October 31, 1957) is an American musician and singer-songwriter who is the leader and creative force behind indie rock group Guided by Voices.', ' In addition to his work with Guided by Voices, he continues to have a prolific solo career with 22 solo albums released so far.']], ['The Crawling Distance', ['The Crawling Distance is 11th studio album released by singer-songwriter Robert Pollard on January 20, 2009.', ' Similar to many of Pollard\\'s releases since \"Fiction Man\" in 2004, all instrumentation on the album was performed by producer Todd Tobias.', ' \"The Crawling Distance\" has a 64/100 score on metacritic and thus was Pollard\\'s lowest rated album on the site, until 2011\\'s \"Space City Kicks\" which has a 62.', ' ']], ['Rob Parissi', ['Robert \"Rob\" Parissi is an American singer, songwriter and guitarist, perhaps best known as frontman for the American funk group Wild Cherry, best known for their 1976 Parissi-penned chart-topper \"Play That Funky Music\".', ' He was born in 1950 and raised in the steel mill town of Mingo Junction, Ohio.', ' He graduated from Mingo High School in 1968.', ' Rob formed the band Wild Cherry in 1970 in Steubenville, Ohio, one mile north of Mingo Junction along the Ohio River.', ' The band played the Ohio Valley region, Wheeling, West Virginia and the rest of the Northern West Virginia panhandle, and Pittsburgh, Pennsylvania.']], ['Choreographed Man of War', ['Choreographed Man of War is an album by Robert Pollard and the Soft Rock Renegades, released in 2001.', ' The album features Robert Pollard (vocals, guitar), Greg Demos (bass), and Jim Macpherson (drums).']], ['Elephant Jokes', [\"Elephant Jokes is the 12th studio album released by singer-songwriter Robert Pollard on August 11, 2009, and the 8th full-length album to be released by Pollard (along with several EP's and singles) since the break-up of his band Guided by Voices in 2004.\", ' Unlike recent Pollard albums, Todd Tobias does not play all the instruments on \"Elephant Jokes\", as Pollard plays some guitar on this album.']], ['Weatherman and Skin Goddess', ['Weatherman and Skin Goddess is a limited EP from singer-songwriter Robert Pollard.', \" Only 1,000 CDs and 500 12 inch LPs were put into production and were made available exclusively on Pollard's website.\", \" Released on April 15, this marks the first release from Robert Pollard's record label Guided by Voices Inc.\"]], ['Kid Marine', ['Kid Marine is 3rd album by Robert Pollard, released in 1999.', \" It is the first release of Robert Pollard's Fading Captain Series.\", ' Pollard has stated that the album is about Jeff \"Kid Marine\" Davis, the person pictured on the cover .', ' Robert told Mojo magazine, \"My personal favorite, a weird record, almost a concept album, about the typical Ohio male and what he does - drink, watch television, eat pizza.', \" It got mixed reviews, there are people who hate it and others who think it's our best record and I'm on their side.\", ' I just love the songs.', ' It feels like one piece, like it all fits together.', ' I like the cover and I like the']], ['Robert Pollard Is Off to Business', ['Robert Pollard Is Off to Business is 10th studio album released by singer-songwriter Robert Pollard on June 2, 2008.', ' This is the first LP release from Robert Pollard\\'s new record label \"Guided by Voices Inc\".', ' All instrumentation on the album was performed by producer Todd Tobias.', ' Many of the songs on the album were over three minutes in length, which is unusual for a Pollard release.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.748\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac245015542992f1f2b3829', 'answer': 'Louis \"Louie\" Zamperini', 'question': 'Who was a Christian Evangelist and US prisoner of war survivor that was the basis for a film directed by Angelina Jolie?', 'supporting_facts': [['Unbroken (film)', 1], ['Louis Zamperini', 0]], 'context': [['Cyborg 2', ['Cyborg 2, released in some countries as Glass Shadow, is a 1993 American science fiction action film directed by Michael Schroeder and starring Elias Koteas, Angelina Jolie, Billy Drago, Karen Sheperd and Jack Palance.', ' It is an unrelated sequel to the 1989 film \"Cyborg\", although footage from the original is used in a dream sequence.', ' It was also Jolie\\'s film debut in a starring role (she had previously made an earlier film, \"Lookin\\' to Get Out\", as a child actress).', ' It was followed by the 1995 direct-to-video release \"\".']], ['Unbroken (film)', ['Unbroken is a 2014 American war film produced and directed by Angelina Jolie, written by the Coen brothers, Richard LaGravenese, and William Nicholson, based on the 2010 non-fiction book by Laura Hillenbrand, \"\".', ' The film revolves around the life of USA Olympian and army officer Louis \"Louie\" Zamperini.', ' Zamperini survived in a raft for 47 days after his bomber crash landed in the ocean during World War II, then was sent to a series of prisoner of war camps.']], ['Salt (2010 film)', ['Salt is a 2010 American action thriller film directed by Phillip Noyce, written by Kurt Wimmer, and starring Angelina Jolie, Liev Schreiber, Daniel Olbrychski, August Diehl, and Chiwetel Ejiofor.', ' Jolie plays Evelyn Salt, who is accused of being a Russian sleeper agent and goes on the run to try to clear her name.']], ['Angelina Jolie filmography', ['Angelina Jolie is an American actress and filmmaker.', ' As a child, she made her screen debut in the 1982 comedy film \"Lookin\\' to Get Out\", acting alongside her father Jon Voight.', ' Eleven years later she appeared in her next feature, the low-budget film \"Cyborg 2\", a commercial failure.', ' She then starred as a teenage hacker in the 1995 science fiction thriller \"Hackers\", which went on to be a cult film despite performing poorly at the box-office.', ' Jolie\\'s career prospects improved with a supporting role in the made-for-television film \"George Wallace\" (1997), for which she received the Golden Globe Award for Best Supporting Actress – Television Film.', ' She made her breakthrough the following year in HBO\\'s television film \"Gia\" (1998).', ' For her performance in the title role of fashion model Gia Carangi, she won the Golden Globe Award for Best Actress – Television Film.']], ['Gone in 60 Seconds (1974 film)', ['Gone in 60 Seconds is a 1974 American action film written, directed, produced by, and starring H.B. \"Toby\" Halicki.', ' It centers on a group of car thieves and the 48 cars they must steal in a matter of days.', ' The film is known for having wrecked and destroyed 93 cars in a 40-minute car chase scene.', ' This film is the basis for the 2000 remake starring Nicolas Cage and Angelina Jolie.']], ['In the Land of Blood and Honey', ['In the Land of Blood and Honey is a 2011 American war film written, produced, and directed by Angelina Jolie and starring Zana Marjanović, Goran Kostić, and Rade Šerbedžija.', \" The film, Jolie's first commercial release as a director, depicts a love story set against the background of the Bosnian War.\", ' It opened in the United States on December 23, 2011, in a limited theatrical release.']], ['By the Sea (2015 film)', ['By the Sea is a 2015 American romantic drama film written and directed by Angelina Jolie, and produced by and starring Jolie and Brad Pitt.', ' The film was released on November 13, 2015, by Universal Pictures.']], ['Angelina Jolie', ['Angelina Jolie Pitt ( ; née Voight; born June 4, 1975) is an American actress, filmmaker, and humanitarian.', \" She has received an Academy Award, two Screen Actors Guild Awards, and three Golden Globe Awards, and has been cited as Hollywood's highest-paid actress.\", ' Jolie made her screen debut as a child alongside her father, Jon Voight, in \"Lookin\\' to Get Out\" (1982).', ' Her film career began in earnest a decade later with the low-budget production \"Cyborg 2\" (1993), followed by her first leading role in a major film, \"Hackers\" (1995).', ' She starred in the critically acclaimed biographical cable films \"George Wallace\" (1997) and \"Gia\" (1998), and won an Academy Award for Best Supporting Actress for her performance in the drama \"Girl, Interrupted\" (1999).']], ['First They Killed My Father (film)', ['First They Killed My Father (Khmer: មុន\\u200bដំបូង\\u200bខ្មែរ\\u200bក្រហម\\u200bសម្លាប់\\u200bប៉ា\\u200bរបស់\\u200bខ្ញុំ \"Moun\\u200b dambaung\\u200b Khmer\\u200b Krahm\\u200b samleab\\u200b ba\\u200b robsa\\u200b khnhom\") is a 2017 biographical historical thriller film directed by Angelina Jolie and written by Jolie and Loung Ung, based on Ung\\'s memoir of the same name.', ' Set in 1975, the film depicts 5-year-old Ung who is forced to be trained as a child soldier while her siblings are sent to labor camps during the Khmer Rouge regime.']], ['Louis Zamperini', ['Louis Silvie \"Louie\" Zamperini (January 26, 1917 – July 2, 2014) was a US prisoner of war survivor in World War II, a Christian evangelist and an Olympic distance runner.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.749\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab7ee895542995dae37e9f6', 'answer': 'Venice', 'question': 'What is the happiest town Sarasota county Florida?', 'supporting_facts': [['Plantation, Sarasota County, Florida', 2], ['Venice, Florida', 4]], 'context': [['Stevens-Gilchrist House', ['The Stevens-Gilchrist House, at 235 Delmar Avenue in Whitfield, Manatee County, Florida, is located in the Whitfield Estates Subdivision in the Sarasota metropolitan area, and was built in 1926.', ' It has also been known as Norrie House.', ' Although the Whitfield Estates Subdivision is in Manatee County, Florida, not in the city of Sarasota, Florida (in Sarasota County) proper, residents use \"Sarasota\" as their mailing address and have associated themselves more with Sarasota, just to the south, rather than with Bradenton a bit further to the north.']], ['Sarasota, Florida', ['Sarasota ( ) is a city in Sarasota County on the southwestern coast of the U.S. state of Florida.', ' The area is renowned for its cultural and environmental amenities, beaches, resorts, and the Sarasota School of Architecture.', ' The city is at the southern end of the Tampa Bay Area, north of Fort Myers and Punta Gorda.', ' Its official limits include Sarasota Bay and several barrier islands between the bay and the Gulf of Mexico.', ' According to the U.S. Census Bureau, in 2013 Sarasota had a population of 53,326.', ' In 1986 it became designated as a certified local government.', ' Sarasota is a principal city of the Sarasota metropolitan area, and is the seat of Sarasota County.']], [\"Florida's 16th congressional district\", [\"Florida's 16th congressional district is an electoral district for the U.S. Congress and was reassigned in 2012, effective January 2013, to western Manatee County, Florida and Sarasota County.\", \" The district stretches from Bradenton, the County Seat, in Manatee County to North Port, in Sarasota County, the county's youngest and most populous incorporated city.\", ' The city of Sarasota is the County Seat of Sarasota County.']], ['Plantation, Sarasota County, Florida', ['Plantation is a census-designated place (CDP) in Sarasota County, Florida, United States.', ' The population was 4,919 at the 2010 census.', ' It is part of the Bradenton–Sarasota–Venice Metropolitan Statistical Area.']], [\"Sarasota County Sheriff's Office\", [\"Sarasota County Sheriff's Office (SSO) is the primary law enforcement agency for Sarasota County, Florida.\", \" The agency is responsible for law enforcement services in unincorporated areas of Sarasota County (home to over 60 percent of the county's residents), jail facilities and courthouse security for Florida's 12th Judicial Circuit.\", \" SSO also operates Public Safety Communications (PSC), the county's primary 911 center.\"]], ['Sarasota County Area Transit', ['Sarasota County Area Transit (SCAT) provides public transportation for Sarasota County, Florida and is operated by the county.', ' SCAT maintains 24 fixed-line bus routes plus a dial-a-ride paratransit service (SCAT Plus).', ' Bus service is offered throughout Sarasota County from 5am until midnight 7 days a week.', ' There is no service on most major holidays, with the Longboat Trolley being the only exception.', ' However, trolley service does not run on Thanksgiving or Christmas Day.']], ['Dwight James Baum', ['Dwight James Baum (1886–1939) was an American architect most active in New York and in Sarasota, Florida.', \" His work includes Cà d'Zan, the Sarasota Times Building (1925), Sarasota County Courthouse (1926), early residences in Temple Terrace, Florida, Sarasota County Courthouse (1927), Pinecroft, West Side YMCA on 63rd Street between Central Park and Columbus Avenue, Columbus Circle (Syracuse, NY) (1934) and Hendricks Memorial Chapel.\"]], ['Sarasota–Bradenton International Airport', ['Sarasota–Bradenton International Airport (IATA: SRQ,\\xa0ICAO: KSRQ,\\xa0FAA LID: SRQ) is in Sarasota County (terminal) and Manatee County (airfield), Florida.', ' Owned by the Sarasota Manatee Airport Authority, it is three miles north of Sarasota (Sarasota County) and six miles south of Bradenton (Manatee County).']], ['Venice, Florida', ['Venice is a city in Sarasota County, Florida, United States.', ' The city includes what locals call \"Venice Island\", a portion of the mainland that is accessed via bridges over the artificially created Intracoastal Waterway.', ' The city is located south of Nokomis and north of Englewood.', ' As of the 2010 census, the city had a population of 20,748.', ' It is noted for its large snowbird population and was voted as a top 10 Happiest Seaside Towns by Coastal Living.']], ['New Braves Spring Training Stadium', ['The Atlanta Braves of Major League Baseball are planning to move to a new Spring Training stadium in Sarasota County, Florida, for the 2019 season.', ' Their lease at Champion Stadium expires at the end of the 2018 season.', ' The ballpark will be located in North Port, Florida in the southern part of Sarasota County, 35 miles south of Sarasota, Florida.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.750\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adc65e85542996e68525350', 'answer': 'no', 'question': 'Are both Dafeng District and Dazhou located in the same province?', 'supporting_facts': [['Dafeng District', 0], ['Dazhou', 0]], 'context': [['Sichuan–Shanghai gas pipeline', ['Sichuan–Shanghai gas pipeline () is a 1702 km long natural gas pipeline in China.', ' The pipeline runs from Pugang gas field in Dazhou, Sichuan Province, to Qingpu District of Shanghai.', ' An 842 km long branch line connects Yichang in Hubei with Puyang in Henan Province.', ' Two shorter branches are located near the Puguang gas field and one in the east near Shanghai.']], ['Yandu District', ['Yandu District () is one of three districts of Yancheng, Jiangsu province, China.', ' (The other two are Tinghu District and Dafeng District).']], ['Dazhou', ['Dazhou () is a prefecture-level city in the northeast corner of Sichuan province, China, bordering Shaanxi to the north and Chongqing to the east and south.', ' 2002 population was 384,525.']], ['Dafeng District', ['Dafeng () is a coastal district under the administration of Yancheng, Jiangsu province, China.', ' Located on the Jiangsu North Plain with a coastline of 112 km , Dafeng was historically one of the largest salt-making areas in China and now is famed for its well preserved eco-system and numerous national conservation parks.', \" The district has the largest national nature reserve for a rare deer species, Père David's Deer or Milu (麋鹿 ) in Chinese.\", ' It borders the prefecture-level city of Taizhou to the southwest.']], ['Tinghu District', ['Tinghu District () is one of three districts of Yancheng, Jiangsu province, China.', ' (The other two are Yandu District and Dafeng District).', ' Prior to 2004, Tinghu District was called the Urban District ()of Yancheng.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.750\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8de7c65542995085b37353', 'answer': 'General Motors', 'question': 'Opel once owned and manufactured the brand known under one shared name in New Zealand and Australia, and that brand is now a subsidiary of what company?', 'supporting_facts': [['Opel', 0], ['Opel', 5], ['Holden', 0], ['Holden', 3]], 'context': [['Opel', ['Opel Automobile GmbH (Opel, ] ) is a German automobile manufacturer, a subsidiary of the French automobile manufacturer Groupe PSA since 1 August 2017.', ' In March 2017, Groupe PSA agreed to acquire Opel from General Motors.', ' The acquisition was approved by the European Commission regulatory authorities in July 2017.', \" Opel's headquarters are in Rüsselsheim am Main, Hesse, Germany.\", ' The company designs, engineers, manufactures and distributes Opel-branded passenger vehicles, light commercial vehicles, and vehicle parts for distribution in Africa, Asia, Europe, and South America.', ' Opel designed and manufactured vehicles are also sold under the Vauxhall brand in Great Britain, the Buick brand in the United States, Canada, Mexico, and China and the Holden brand in Australia and New Zealand.']], ['Opel Cascada', ['The Opel Cascada (Spanish for \"waterfall\") is a Mid-size convertible engineered and manufactured by the German automaker Opel since 2013.', ' It is also marketed as the Opel Cabrio in Spain, Vauxhall Cascada in the United Kingdom, the Holden Cascada in Australia and New Zealand, and the Buick Cascada in the United States and China.']], ['Tennis New Zealand', ['The history of tennis in New Zealand dates back to the 1870s, the decade when the development of modern tennis began.', ' The first \"New Zealand Tennis Championships\" were played at Farndon in Hawkes Bay in 1886.', ' New Zealand Lawn Tennis Association (NZLTA) was formed at a meeting held in Hastings in December 1886.', ' Shortly after its inauguration, the New Zealand Association became affiliated with the Lawn Tennis Association (England).', ' In 1904 New Zealand Lawn Tennis Association amalgamated with six Australian state tennis associations to form the Lawn Tennis Association of Australasia.', ' New Zealand Lawn Tennis Association played a significant role in the origin of the Australian Open.', ' Lawn Tennis Association of Australasia created the tournament called \"The Australasian Mens Championships\" (which later became Australian Open) in 1905 and was first played in Warehouseman\\'s Cricket Ground and it was decided that championships would be hosted by both Australian as well as New Zealand venues.', ' New Zealand hosted the championship twice— Christchurch (1906) and Hastings (1912).', ' The geographical remoteness of both the countries (Australia and New Zealand) made it difficult for foreign players to enter the tournament.', ' In Christchurch in 1906, of a small field of 10 players, only two Australians attended, and the tournament was won by a New Zealander (Tony Wilding).', ' Lawn Tennis Association of Australasia was one of the twelve national associations of tennis which established the International Lawn Tennis Federation (ILTF) in a conference in Paris, France on 1 March 1913.', ' From 1905 until 1919, New Zealand and Australian tennis players participated in the International Lawn Tennis Challenge (Davis Cup) under the alias of \"Team Australasia\", the team claimed a title six times (1907, 1908, 1909, 1911, 1914, 1919), however, there were attempts to severance this trans-tasmanian partnership, in order to allow New Zealand players to represent their nation on international tennis events.', ' In 1922, New Zealand dropped out from this partnership and on 16 March 1923 New Zealand Lawn Tennis Association was granted affiliation to the International Lawn Tennis Association and thereby became eligible to enter the International Lawn Tennis Challenge in its own right.', ' New Zealand Lawn Tennis Association filed its first challenge with United States Lawn Tennis Association for 1924 International Lawn Tennis Challenge.', ' Tennis New Zealand was the founding member of Oceania Tennis Federation in 1993.']], ['Ansett New Zealand', ['Ansett New Zealand was a wholly owned airline subsidiary of Ansett Australia, serving the New Zealand domestic market between 1987 and 2000.', ' In order to comply with regulatory requirements relating to the acquisition of Ansett Australia by Air New Zealand, Ansett New Zealand was sold to News Corporation and later to Tasman Pacific Airlines of New Zealand in 2000, operating as a Qantas franchise under the Qantas New Zealand brand.', ' It went into receivership and subsequently liquidation in 2001.']], ['Holden', ['Holden, formally known as General Motors Holden, is an Australian automobile manufacturer with its headquarters in Port Melbourne, Victoria.', ' The company was founded in 1856 as a saddlery manufacturer in South Australia.', ' In 1908 it moved into the automotive field, before becoming a subsidiary of the United States-based General Motors (GM) in 1931.', \" After becoming a subsidiary of GM, the company was named General Motors-Holden's Ltd, becoming Holden Ltd in 1998 and General Motors Holden in 2005.\"]], ['NZI', ['NZI or New Zealand Insurance is a major insurance company in New Zealand.', \" NZI was formed in Auckland in 1859 as the New Zealand Insurance Company Ltd and is one of New Zealand's largest and longest-serving fire and general insurance brands.\", ' It merged with South British Insurance in 1981; the two companies had been equally matched rivals and were virtually the same size, but with different emphasis on the types of business they held.', ' The new company formed a parent, New Zealand South British Group Ltd, which maintained both brands concurrently before changing to the NZI Corporation in 1984 when the South British brand was phased out.', \" In January 2003 IAG, Insurance Australia Group, purchased NZI when acquiring Aviva's general insurance business, and NZI is now a subsidiary of IAG New Zealand Ltd.\", ' Aviva predecessor General Accident bought NZI in 1989.', ' NZI focuses on providing products to the intermediated market; i.e. brokers and banks.']], ['Jetconnect', ['Jetconnect is a wholly owned subsidiary airline of Qantas that is based in Auckland, New Zealand.', ' It was established in July 2002, commencing operations in October the same year.', ' It operates trans-Tasman services between New Zealand and Australia under the Qantas brand.', ' It employs crew based in New Zealand and operates aircraft registered in New Zealand.', ' It also operated domestic services within New Zealand until these services were taken over by Jetstar Airways, another Qantas subsidiary, on 10 June 2009.', ' Its main base is Auckland Airport.']], ['ANZ Bank New Zealand', [\"ANZ Bank New Zealand Limited, New Zealand's largest financial-services group, operates as a subsidiary of Australia and New Zealand Banking Group Limited of Australia.\", ' Until 2012, ANZ operated in New Zealand under the legal entity ANZ National Bank Limited, which was formed as part of the 2012 merger of ANZ Banking Group (New Zealand) Limited and the National Bank of New Zealand Limited.', ' From 2012, the company was renamed ANZ Bank New Zealand as part of the merger of ANZ and the National Bank brands.', ' ANZ New Zealand operates under a variety of different brands, such as ANZ, UDC Finance, Bonus Bonds and Direct Broking.', ' It provides a number of financial services, including banking services, asset finance, investments and payment \"solutions\".']], ['The Willy Wonka Candy Company', ['The Willy Wonka Candy Company is a British brand of confectionery owned and licensed by Swiss corporation Nestlé.', \" The Wonka brand's inception comes from materials licensed from British author Roald Dahl.\", ' His classic children\\'s novel, \"Charlie and the Chocolate Factory\", and its film adaptations are the source of both the packaging and the marketing styles of the Wonka brand.', \" The brand was launched in 1971, coinciding with the release of the novel's first film adaptation.\", ' In 1988 the Willy Wonka Candy Company brand – then owned by Sunmark Corporation – was acquired by Nestlé.', ' Nestlé sells sweets and chocolate under the Willy Wonka brand name in the United States, Canada, the United Kingdom, the Republic of Ireland, Australia, New Zealand, Japan, South Africa, Mexico, Colombia, Brazil, Argentina, Costa Rica, Panama, Dominican Republic and the Middle East.', ' In mid-2015 the Willy Wonka brand name was dropped by Nestlé, in favour of special \"throwback\" packaging.', ' Candies previously made by the Willy Wonka brand are now under the Nestlé brand naming, excluding the Wonka brand name on the top left corner.']], ['Holden Astra', ['The Holden Astra is a compact car marketed by Holden in Australia.', ' Spanning six generations, the original, Australia-only Astra of 1984 was a derivative of the locally produced Nissan Pulsar, as was the 1987 Astra.', ' It was succeeded by the Holden Nova in 1989—another unique-to-Australia model line.', ' From 1995, the Holden Astra name was used in New Zealand, for a badge engineered version of the Opel Astra, which had been sold locally as an Opel since 1993.', ' The following year, Holden discontinued the Nova line in Australia in favour of the Opel-based Holden Astra.', ' On 1 May 2014, Holden announced to import the Opel Astra J GTC and Opel Astra J OPC with Holden badges to Australia and New Zealand.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.751\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac54ca45542993e66e822d5', 'answer': 'The Blue Album', 'question': 'What was the title of the debut studio album by Weezer which contained the hit Buddy Holly and Undone – The Sweater Song?', 'supporting_facts': [['Weezer discography', 1], ['Buddy Holly (song)', 1]], 'context': [['Closing In on the Fire', ['Closing In on the Fire is an album by Waylon Jennings, released on the small Ark 21 Records label on June 16, 1998.', ' It features contributions from several celebrities associated with both country and rock music.', ' The selections include, among others, Sting\\'s \"She\\'s Too Good for Me\" and Tony Joe White\\'s title track, creating a greater degree of musical eclecticism than in many previous Jennings releases.', ' The singer incorporated elements of genres such as blues and rock, in addition to traditional country ballads.', ' \"Best Friends of Mine\", an autobiographical song, is a tribute to Buddy Holly, Hank Williams, Jr. and one of Jennings\\' close friends from his days in Phoenix.', ' Carl Smith, one of the performer\\'s idols, appears on \"Untitled Waltz\".', ' In an interview, the singer mentioned that he wasn\\'t fully satisfied with his take on The Rolling Stones\\' \"No Expectations\", calling it \"\"a little more contrived than I would have liked\"\".', ' An interview featuring Jennings commenting on the record is included as a bonus track.', ' \"Closing In on the Fire\", Jennings\\' 72nd release, reached #71 on the country charts and was the last studio album by the singer to be released before his death in 2002.']], ['True Love Ways', ['\"True Love Ways\" is a song written by Buddy Holly and Norman Petty and recorded with the Dick Jacobs Orchestra in October 1958, four months before the singer\\'s death.', ' Some argue that this song is the most played \"first song\" at weddings.', ' It was first released on the posthumous album \"The Buddy Holly Story, Vol.', ' 2\" (Coral 57326/757326), in March 1960.', ' The song was a hit in Britain in 1960, reaching number 25 on the pop singles chart.']], ['Buddy Holly (album)', ['Buddy Holly is a studio album by Buddy Holly.', ' It was released by Coral Records on February 20, 1958.', ' The album collects Holly\\'s four hit singles released on the Coral label; \"Words of Love\", \"Peggy Sue\", \"I\\'m Gonna Love You Too\", and \"Rave On!', '\".', \" The backing group was Buddy Holly's current band, the Crickets.\"]], [\"That'll Be the Day (album)\", [\"That'll Be The Day is the final studio album from Buddy Holly.\", ' Decca, Holly’s first major record label, after failing to produce a hit single from Holly’s early recordings, packaged these 1956 tunes after he had some success with recordings from the Brunswick and Coral labels, i.e. the previously released single \"That\\'ll Be the Day\".', ' This is the last album released before his death in a plane crash on February 3, 1959, and is rare among collectors.']], ['Undone – The Sweater Song', ['\"Undone – The Sweater Song\" is a song by the American alternative rock band Weezer, released on the band\\'s self-titled 1994 debut album.', ' It was released as their debut single in 1994.', ' Weezer frontman Rivers Cuomo has commented on the song, saying:']], ['Sweater Weather', ['\"Sweater Weather\" is a song by American indie rock band The Neighbourhood.', ' The song was written by group members Jesse Rutherford, Zach Abels and Jeremy Freedman, and was produced by Justyn Pilbrow.', ' It serves as the lead single from their debut studio album, \"I Love You\" (2013).', ' \"Sweater Weather\" reached number one on the \"Billboard\" Alternative Songs chart in June 2013, logging eleven non-consecutive weeks at the summit of the chart.', ' It was re-released on November 2, 2013 in honor of the 2013 winter season.']], ['Weezer (1994 album)', ['Weezer, also known as the Blue Album, is the eponymous debut studio album by American rock band Weezer, released on May 10, 1994 through DGC Records.', ' The album was produced by The Cars frontman Ric Ocasek and recorded in Electric Lady Studios in New York City.', ' The Blue Album was supported by three singles: \"Undone – The Sweater Song\", \"Buddy Holly\" and \"Say It Ain\\'t So\", which brought Weezer mainstream success, helped by music videos directed by Spike Jonze.']], ['Rave On Buddy Holly', ['Rave On Buddy Holly is a compilation album by various artists released on June 28, 2011, through Fantasy Records/Concord Music Group and Hear Music.', ' A tribute album to musician Buddy Holly, who died in a plane crash in 1959 at age 22, the title refers to the song \"Rave On\", one of his biggest hits.', \" Contributing artists included Paul McCartney, who owned Holly's publishing catalog at the time of the album's release, and Graham Nash, a former member of The Hollies, who were named in commemoration of Holly.\"]], ['Weezer discography', ['The discography of Weezer, an American rock band, consists of 10 studio albums, two compilation albums, one video album, six extended plays, twenty-eight singles and twenty-four music videos. Weezer\\'s self-titled debut studio album, often referred to as \"The Blue Album\", was released in May 1994 through DGC Records.', ' The album was a huge commercial success, peaking at number 16 on the US \"Billboard\" 200 and spawning the singles \"Undone – The Sweater Song\" and \"Buddy Holly\", both of which were responsible for launching Weezer into mainstream success with the aid of music videos directed by Spike Jonze.', \" It has sold 3.3 million copies in the United States and has been certified triple platinum by the Recording Industry Association of America (RIAA), becoming the band's best selling album to date.\", ' Following the success of their debut album, Weezer took a break from touring for the Christmas holidays.', \" Lead singer Rivers Cuomo began piecing together demo material for Weezer's second studio album.\", ' Cuomo\\'s original concept for the album was a space-themed rock opera, \"Songs from the Black Hole\".', ' Ultimately, the \"Songs from the Black Hole\" album concept was dropped; the band, however, continued to utilize songs from these sessions into work for their second studio album. \"', 'Pinkerton\" was released as the band\\'s second studio album in September 1996.', ' Peaking at number 19 on the \"Billboard\" 200, it was considered a critical and commercial failure at the time of its release, selling far less than its triple platinum predecessor.', ' However, in the years following its release, it has seen much critical and commercial championing.']], ['Buddy Holly (song)', ['\"Buddy Holly\" is a song by the American rock band Weezer, written by Rivers Cuomo.', ' It was released as the second single from the band\\'s debut album \"Weezer\" (\"The Blue Album\") in 1994.', \" The single was released on what would have been Buddy Holly's 58th birthday.\", \" The lyrics reference the song's 1950s namesake and actress Mary Tyler Moore.\", ' It reached #2 and #34 on the US Modern Rock Tracks chart and the US Mainstream Rock Tracks chart, respectively.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.752\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8b5c8155429949d91db56d', 'answer': 'Salford, Lancashire', 'question': 'Where is the 1999 British comedy-drama film starring Jimi Mistry set?', 'supporting_facts': [['Jimi Mistry', 0], ['East Is East (1999 film)', 1]], 'context': [['Festival of Lights (film)', ['Festival of Lights is a 2010 film directed and written by Shundell Prasad.', \" It stars Melinda Shankar as the rebellious and mouthy Reshma, Jimi Mistry as Reshma's birth father, and Aidan Quinn as Reshma's stepfather.\", \" The film deals with Reshma's struggles to find her father whom she was separated from when she and her mother, played by Ritu Singh Pande, migrate from Guyana to New York City.\"]], ['RocknRolla', ['RocknRolla is a 2008 British-American crime comedy film written and directed by Guy Ritchie, and starring Gerard Butler, Tom Wilkinson, Thandie Newton, Mark Strong, Idris Elba, Tom Hardy, Jimi Mistry and Toby Kebbell.', ' It was released on 5 September 2008 in the UK, hitting No. 1 in the UK box office in its first week of release.']], ['Partition (2007 film)', ['Partition is a 2007 film directed by Vic Sarin, written by Patricia Finn and Vic Sarin, and starring Jimi Mistry and Kristin Kreuk.', ' The film is set in 1947, based on the partition of India and was partially shot in Kamloops, British Columbia, Canada.']], ['East Is East (1999 film)', [\"East Is East is a 1999 British comedy-drama film written by Ayub Khan-Din and directed by Damien O'Donnell.\", ' It is set in Salford, Lancashire, in 1971, in a mixed-ethnicity British household headed by Pakistani father George (Om Puri) and an English mother, Ella (Linda Bassett).']], ['The Truth About Love (film)', ['The Truth About Love is a 2005 film directed by John Hay and starring Jennifer Love Hewitt, Jimi Mistry and Dougray Scott.']], ['Exam (2009 film)', ['Exam is a 2009 British psychological thriller film written by Simon Garrity and Stuart Hazeldine, directed by Hazeldine, and starring Colin Salmon, Chris Carey, Jimi Mistry, Luke Mably, Gemma Chan, Chuk Iwuji, John Lloyd Fillingham, Pollyanna McIntosh, Adar Beck and Nathalie Cox.']], ['Jimi Mistry', ['Jimi Mistry (born 1 January 1973) is an English actor, best known for his roles in \"EastEnders\" and \"Coronation Street\" as well as appearing in numerous films such as \"East Is East\", \"Blood Diamond\", \"The Guru\", \"Exam\", \"West is West\", \"Ella Enchanted\" and \"The Truth About Love\".']], ['West Is West (2010 film)', ['West Is West is a 2010 British comedy-drama film, which is a sequel to the 1999 comedy \"East Is East\".', ' It stars Om Puri, Linda Bassett, Aqib Khan, Ila Arun and Jimi Mistry, is written by Ayub Khan-Din, directed by Andy DeEmmony, and produced by Leslee Udwin for Assassin Films and BBC Films.']], ['My Kingdom (film)', ['My Kingdom is a 2001 British crime film directed by Don Boyd and starring Richard Harris, Lynn Redgrave and Jimi Mistry.']], ['Fawad Siddiqui', ['Fawad Siddiqui is an American actor, improvisational comedian, journalist and cartoonist.', ' He has appeared on the USA Network television show \"Burn Notice\" alongside Bruce Campbell and Jeffrey Donovan, in the George Clooney film \"The Men Who Stare At Goats\" with Ewan McGregor [1], and in the indie films \"The Bait\" and \"Festival of Lights\"—starring Jimi Mistry and Aidan Quinn.', ' He also had a role in Queen Latifah produced sequel The Cookout Part 2—starring Charlie Murphy, Mike Tyson and Faizon Love—and currently has a recurring role on the fifth season of the Lifetime Network series \"Army Wives\".', ' He also appeared in the latest season of the FX/The Audience Network series \"Damages\"—starring Glenn Close, Rose Byrne and John Goodman—as the shady Afghan information broker Shahbaz Gul opposite Dylan Baker, and in the French comedy \"Bienvenue à Bord\"—opposite popular French comedic actors Franck Dubosc and Valérie Lemercier, released in October 2011.', ' He will be featured in the upcoming 2013 indie drama \"Sunlight Jr.\", from critically acclaimed director Laurie Collyer, opposite Matt Dillon and Naomi Watts.', ' And he played the role of Mohammed Al Ghamdi in the second season of the Emmy Award-winning Showtime series Homeland, starring Claire Danes and Mandy Patinkin.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.753\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a74547755429979e2882900', 'answer': 'Texas A&M Aggies football', 'question': 'the head football coach at the University of Houston from 2007 to 2011, is the current team coach of which football team ?', 'supporting_facts': [['Texas A&amp;M Aggies football', 0], ['Texas A&amp;M Aggies football', 4], ['Kevin Sumlin', 1]], 'context': [['Willie Fritz', ['Willie Fritz (born April 2, 1960) is an American football coach and former player.', ' He is the current head coach at Tulane University.', ' From 2014 to 2015, he was head coach at Georgia Southern University.', ' From 2010 to 2013, he was the head football coach at Sam Houston State University.', ' From 1997 to 2009, Fritz served as the head football coach at the University of Central Missouri.', ' From 1993 to 1996, he was the head football coach at Blinn College, a junior college in Brenham, Texas.']], ['Carl Anderson (American football)', ['Carl Rudolph Frederick \"Swede\" Anderson IV (September 9, 1898 – April 30, 1978) was an American college football coach at Western Kentucky University and Howard Payne University.', ' Anderson graduated from Centre College in Danville, Kentucky in 1924, where he played in the backfield with legendary alumnus Bo McMillin.', ' Anderson then followed McMillin to Centenary College of Louisiana and Geneva College.', ' Anderson then served one year as the head football coach at Western Kentucky, before moving to Kansas State as its freshman team coach in 1930.', ' Anderson returned to Western Kentucky as its head coach from 1934 to 1937.', ' He was the backfield coach under McMillin at Indiana from 1938 to 1945.', ' He then returned to his alma mater, Centre College, where he coached the Praying Colonels until 1950.', ' The following season, Anderson became the seventh head football coach at the Howard Payne University in Brownwood, Texas and held that position from 1951 to 1952.', ' His coaching record at Howard Payne was 7–10.']], ['Tom Keele', ['Tom Keele (born c. 1933) is a former American football coach.', ' He served as the head football coach at California State University, Northridge from 1979 to 1985, compiling a record of 31–42–1.', ' Keele graduated from Jefferson High School in Portland Oregon in 1951.', ' He attended the University of Oregon, where he played football for the Oregon Webfoots as a tackle from 1957 to 1959.', ' Keele began his coaching career in 1960 at North Eugene High School in Eugene, Oregon, working two years as an assistant football coach and sophomore basketball coach.', ' He moved to Oregon City High School in Oregon City, Oregon in 1962, serving as head football coach and leading his team to a 9–1–1 record.', ' The following year, he was hired as head football coach at the newly-formed Sheldon High School in Eugene.']], ['Tim Landis', ['Timothy Joseph \"Tim\" Landis (born July 13, 1964) is an American football coach who is currently quarterbacks coach and special teams coordinator at Lycoming College.', ' Previously, Landis was the head coach for the Rensselaer Polytechnic Institute football team.', ' He was also formerly the offensive coordinator for the San Jose State Spartans football team and the head football coach for Bucknell University.', ' He compiled a 23–33 record at Bucknell since 2003 and a 76–85–1 record overall.', \" Prior to arriving at Bucknell, Landis served as head football coach at Davidson and St. Mary's.\"]], ['Kevin Sumlin', ['Kevin Warren Sumlin (born August 3, 1964) is an American football coach and former player who is the head coach at Texas A&M University.', ' Previously, Sumlin was the head football coach at the University of Houston from 2007 to 2011.']], ['Robert P. Wilson', ['Robert P. \"Bert\" Wilson was an American football player and coach.', \" He played football for Wesleyan University and was captain of the school's football team in 1896.\", \" After graduating, he served as Wesleyan's first head football coach from 1898 to 1902.\", \" In five years as Wesleyan's coach, Wilson compiled a record of 25–21–2.\", ' In his first two years as the coach, Wesleyan compiled records of 7–3 and 7–2.', \" In the 17 years before Wilson took over as the coach, Wesleyan's football team had never won seven games in a single season.\", ' In 1903, Wilson became the head football coach at New York University (NYU).', ' He served the sixth head football coach at NYU and held that position for one season, in 1903, leading the NYU Violets to a record of 2–5.']], ['Ernest T. Jones', ['Ernest T. Jones (born January 18, 1970) is the current head coach at ASA Miami, a two-year college starting its first football season in 2015.', ' He was briefly running backs coach for the University of Connecticut Huskies football team.', ' He was head football coach at Alcorn State University.', ' He was named the head football coach after the 2007 season and served as head coach in 2008.', ' He was controversially fired from this position in December 2008.', ' He returned to the University of Cincinnati as the Director of Player Services in 2009.', ' For the 2010 he will be an assistant coach at the University at Buffalo under former University of Cincinnati assistant coach and now UB head football Coach Jeff Quinn.']], ['K. C. Keeler', ['Kurt Charles \"K. C.\" Keeler (born July 26, 1959) is an American football coach and former player.', ' He is currently the head football coach at Sam Houston State University.', ' He was the head football coach at the University of Delaware from 2002 to 2012.', ' Keeler served as the head football coach at Rowan University from 1993 to 2001.', \" His 2003 Delaware Fightin' Blue Hens squad won the NCAA Division I-AA Football Championship, and returned to the Division I Championship game in 2007 and 2010.\"]], ['Butch Davis', ['Paul Hilton \"Butch\" Davis, Jr. (born November 17, 1951) is an American football coach.', ' He is the head football coach at Florida International University.', ' After graduating from the University of Arkansas, he became an assistant college football coach at Oklahoma State University and the University of Miami before becoming the defensive coordinator for the Dallas Cowboys of the National Football League (NFL).', \" He was head coach of the University of Miami's Hurricanes football team from 1995 to 2000 and the NFL's Cleveland Browns from 2001 to 2004.\", ' Davis served as the head coach of the University of North Carolina at Chapel Hill (UNC) Tar Heels football team from 2007 until the summer of 2011, when a series of National Collegiate Athletic Association (NCAA) investigations resulted in his dismissal.', \" He was hired by the NFL's Tampa Bay Buccaneers as an advisor in February 2012.\"]], ['Texas A&amp;M Aggies football', ['The Texas A&M Aggies football program represents Texas A&M University in the sport of American football.', ' The Aggies compete in the Football Bowl Subdivision (FBS) of the National Collegiate Athletic Association (NCAA) and the Western Division of the Southeastern Conference (SEC).', ' Texas A&M football claims three national titles and eighteen conference titles.', ' The team plays all home games at the newly redeveloped Kyle Field, a 102,733-person capacity outdoor stadium on the university campus.', \" Kevin Sumlin is currently the team's head coach.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.754\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7c917e5542990527d554d3', 'answer': '42.195', 'question': 'How many kilometers did Sabrina Mockenhaupt run when she set her best record?', 'supporting_facts': [['Sabrina Mockenhaupt', 3], ['Berlin Marathon', 1]], 'context': [['Ellerslie, Edmonton (area)', ['Ellerslie is an area in the southeast portion of the City of Edmonton in Alberta, Canada.', \" It was established in 1999 through Edmonton City Council's adoption of the Ellerslie Area Structure Plan, which guides the overall development of predominantly residential neighbourhoods in the area.\", ' While City of Edmonton documents note that there are diverse stories related to Scottish settlers introducing the name Ellerslie, it is known that the name was applied to the local school district by 1895, and to the local post office in 1896, when it was still a rural area many kilometers south of the early boundaries of the recently (1892) incorporated Town of Edmonton.']], ['Sabrina Mockenhaupt', ['Sabrina Mockenhaupt (born 6 December 1980 in Siegen) is a German long-distance runner who specialises in track events and the marathon.', ' She is a two-time winner of the Cologne Marathon and has also won the Frankfurt Marathon and the Berlin Half Marathon.', ' She represented Germany at the 2004, 2008 Summer Olympics and 2012 Summer Olympics and was the 3000 metres bronze medallist at the 2005 European Indoor Championships.', ' She has a marathon best of 2:26:21, set at the 2010 Berlin Marathon.']], ['Ellerslie, Edmonton', ['Ellerslie is a residential neighbourhood in southeast Edmonton, Alberta, Canada.', ' While City of Edmonton documents note that there are diverse stories related to Scottish settlers introducing the name Ellerslie, it is known that the name was applied to the local school district by 1895, and to the local post office in 1896, when it was still a rural area many kilometers south of the early boundaries of the recently (1892) incorporated Town of Edmonton.']], ['Palmason Model', ['The Palmason Model is a depth, distance, temperature and heat flow gradient model of crustal accretion mechanism through the Iceland lithosphere which denotes the spreading material trajectories from a rift axis.', ' The material erupting at the rift axis will tend to sink down, due to thermal subsidience and spreading, to a depth of many kilometers, while lava flows spreading to a distance of many kilometers away from the rift axis on the surface will sink down to shallower depth.', ' Surface erosion can expose such preserved materials.']], ['Telial stage', ['The Telial stage is one of the stages in the life cycle of a parasitic heteroecious fungus.', ' It is discernible by the formation of large teliospores that the fungi produces to overwinter.', ' The telial stage of heteroecious parasitic fungi is spent on the secondary host plant.', ' A primary aecial stage is spent parasitizing a separate host plant which is a precursor in the life cycle of heteroecious fungi.', ' Spores are released from the telia in the spring.', ' The spores can spread many kilometers through the air, however most are spread near the host plant (Brand, 2004).']], ['Fissure vent', ['A fissure vent, also known as a volcanic fissure or eruption fissure, is a linear volcanic vent through which lava erupts, usually without any explosive activity.', ' The vent is often a few meters wide and may be many kilometers long.', ' Fissure vents can cause large flood basalts which run first in lava channels and later in lava tubes.', ' After some time the eruption builds up spatter resp.', ' ash cones and may concentrate on one or some of them.', ' Small fissure vents may not be easily discernible from the air, but the crater rows (see Laki) or the canyons (see Eldgjá) built up by some of them are.']], ['Siju Cave', ['Siju Cave is located in Meghalaya state near Naphak Lane and Simsang River game reserve.', ' It is a limestone cave.', ' The Cave is filled with water and is many kilometers long.', ' It is considered as the third longest cave system in India.', ' In 1927 it was found that the caves have a temperature of 21–26.4\\xa0°C.']], ['Berlin Marathon', ['The Berlin Marathon (branded BMW Berlin Marathon for sponsorship reasons) is a major running and sporting event held annually in Berlin, Germany.', ' The official marathon distance of 42.195 kilometers (26 miles 385 yards) is set up as a citywide road race where professional athletes and amateur runners jointly participate.', ' Initiated in 1974, the event traditionally takes place on the last weekend in September.']], ['Piercing point', ['In geology, a piercing point is defined as a feature (usually a geologic feature, preferably a linear feature) that is cut by a fault, then moved apart.', ' Reconfiguring the piercing point back in its original position is the primary way geologists can find out the minimum slip, or displacement, along a fault.', ' This can be done on a large scale (over many kilometers), a small scale (inside a single outcrop or fault trench) or even a single hand sample/rock (see image).']], ['Skyway', ['A skyway, skybridge, or skywalk is a type of pedway consisting of an enclosed or covered footbridge between two or more buildings in an urban area.', ' This protects pedestrians from the weather.', ' In North America skyways are usually owned by businesses, and are therefore not public spaces (compare with sidewalk).', \" However, in Asia, such as Bangkok's and Hong Kong's skywalks, they are built and owned separately by the city government, connecting between privately run rail stations or other transport with their own footbridges, and run many kilometers.\", ' Skyways usually connect on the first few floors above the ground-level floor, though they are sometimes much higher, as in Petronas Towers.', ' The space in the buildings connected by skyways is often devoted to retail business, so areas around the skyway may operate as a shopping mall.', ' Non-commercial areas with closely associated buildings, such as university campuses, can often have skyways and/or tunnels connecting buildings.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.754\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adf3f1d5542992d7e9f9310', 'answer': 'Italian composer', 'question': 'What profeesion do Giacomo Benvenuti and Claudio Monteverdi share?', 'supporting_facts': [['Giacomo Benvenuti', 0], ['Claudio Monteverdi', 0]], 'context': [['Giacomo Badoaro', ['Giacomo Badoaro (1602–1654) was a Venetian nobleman and amateur poet.', ' He is most famous for writing the libretto for Claudio Monteverdi\\'s opera \"Il ritorno d\\'Ulisse in patria\" (1640).', ' He also provided librettos for the operas \"Ulisse errante\" by Francesco Sacrati (1644) and \"Elena rapita da Teseo\" (1653) by Jacopo Melani.', ' He was a member of the Venetian intellectual circle, the Accademia degli Incogniti.']], ['Claudio Monteverdi', ['Claudio Giovanni Antonio Monteverdi (] ; 15 May 1567 (baptized) – 29 November 1643) was an Italian composer, string player and choirmaster.', ' A composer of both secular and sacred music, and a pioneer in the development of opera, he is considered a crucial transitional figure between the Renaissance and the Baroque periods of music history.']], ['Giacomo Benvenuti', ['Giacomo Benvenuti (16 March 1885, Toscolano — 20 January 1943, Barbarano-Salò) was an Italian composer and musicologist.', ' He was the son of organist Cristoforo Benvenuti and studied at the Liceo Musicale in Bologna under Luigi Torchi (musicology) and Marco Enrico Bossi (organ).', ' In 1919 his collection of songs for voice and piano accompaniment, \"Canti a una voce : con accompagnamento di pianoforte\", was published in Bologna.', ' In 1922 he published a collection of 17th-century art songs entitled \"35 Arie di vari autori del secolo XVII\".', ' Composer Samuel Barber studied the works of Giulio Caccini, Andrea Falconieri, and other early Italian composers under his tutelage in Milan in 1933-1934.', ' For the Teatro dell\\'Opera di Roma he adapted Claudio Monteverdi\\'s \"L\\'Orfeo\" for a production which premiered on 27 December 1934.', ' The adaptation was later used for the first recording of \"L\\'Orfeo\" in 1939, which included a performance by the orchestra of La Scala Milan under conductor Ferrucio Calusio.']], [\"Il ritorno d'Ulisse in patria\", ['Il ritorno d\\'Ulisse in patria (SV 325, \"The Return of Ulysses to his Homeland\") is an opera consisting of a prologue and five acts (later revised to three), set by Claudio Monteverdi to a libretto by Giacomo Badoaro.', ' The opera was first performed at the Teatro Santi Giovanni e Paolo in Venice during the 1639–1640 carnival season.', ' The story, taken from the second half of Homer\\'s \"Odyssey\", tells how constancy and virtue are ultimately rewarded, treachery and deception overcome.', ' After his long journey home from the Trojan Wars Ulisse, king of Ithaca, finally returns to his kingdom where he finds that a trio of villainous suitors are importuning his faithful queen, Penelope.', ' With the assistance of the gods, his son Telemaco and a staunch friend Eumete, Ulisse vanquishes the suitors and recovers his kingdom.']], [\"L'Orfeo discography\", ['These lists show the audio and visual recordings of the opera \"L\\'Orfeo\" by Claudio Monteverdi.', ' The opera was first performed in Mantua in 1607, at the court of Duke Vincenzo Gonzaga, and is one of the earliest of all operas.', ' The first recording of \"L\\'Orfeo\" was issued in 1939, a freely adapted version of Monteverdi\\'s music edited by Giacomo Benvenuti, given by the orchestra of La Scala Milan conducted by Ferrucio Calusio.', ' In 1949 the Berlin Radio Orchestra under Helmut Koch recorded the complete opera, on long-playing records (LPs).', ' The advent of LP recordings was, as Harold Schonberg later wrote, an important factor in the postwar revival of interest in Renaissance and Baroque music, and from the mid-1950s recordings of \"L\\'Orfeo\" have been issued on many labels.', \" Koch's landmark version was reissued in 1962, when it was compared unfavourably with others that had by then been issued.\", ' The 1969 recording by Nicholas Harnoncourt and the Vienna Concentus Musicus, using Harnoncourt\\'s edition based on period instruments, was praised for \"making Monteverdi\\'s music sound something like the way he imagined\".', ' In 1981 Siegfried Heinrich, with the Early Music Studio of the Hesse Chamber Orchestra, recorded a version which re-created the original Striggio libretto ending, adding music from Monteverdi\\'s 1616 ballet \"Tirsi e Clori\" for the Bacchante scenes.', ' Among more recent recordings, that of Emmanuelle Haïm has been praised for its dramatic effect.', ' The 21st century has seen the issue of an increasing number of recordings on DVD.']], ['John Whenham', ['John Whenham is an English musicologist and academic who specializes in early Italian baroque music.', ' He earned both a Bachelor of Music and a Master of Music from the University of Nottingham, and a Doctor of Philosophy from the University of Oxford.', ' He is a leading expert on the life and works of Claudio Monteverdi, and is the author of the books \"Duet and Dialogue in the Age of Monteverdi\" (Ann Arbor, Michigan: University Microfilms International, 1982) \"Monteverdi, \\'Orfeo\\' \" (London: Cambridge University Press, 1986), \"Monteverdi, Vespers (1610)\" (Cambridge University Press, 1997), and \"The Cambridge Companion to Monteverdi\" (with Richard Wistreich, Cambridge University Press, 2007).', ' For five years he was co-editor of the journal \"Music & Letters\".', ' He currently serves on the board of the Birmingham Early Music Festival and is head of the music history department at the University of Birmingham.']], ['Ricciardo Amadino', ['Ricciardo Amadino (\"fl.\"', ' 1572–1621) was a Venetian printer.', ' He briefly attempted to publish music on his own in 1579, but was unsuccessful.', ' He joined with Giacomo Vincenti, with whom he published over 80 books between 1583 and 1586.', ' Many of these were reprints of popular madrigal books, but some were first printings.', ' Their partnership ended around 1586, but they continued to work together occasionally.', ' After 1586, Amadino\\'s mark was a woodcut of an organ, and he printed primarily music, with a few theoretical treatises, including the first edition of Ercole Bottrigari\\'s \"Il desiderio\".', ' He printed editions of such important composers as Luca Marenzio and Claudio Monteverdi, including the celebrated 1609 edition of \"L\\'Orfeo\", and in terms of sheer output was one of the foremost Italian music printers.']], ['Sergio Vartolo', ['Sergio Vartolo (Bologna, 1944) is an Italian harpsichordist, organist, musicologist and conductor; in past also active as countertenor.', ' In 1996 he was appointed maestro de capella of the Cappella Musicale di San Petronio di Bologna founded in 1436.', ' He has an extensive discography, both as a harpsichordist - the complete works of Girolamo Frescobaldi, and as a conductor - particularly works by Giovanni Paolo Colonna and Giacomo Antonio Perti associated with San Petronio, but also operas by Claudio Monteverdi and others.']], ['Stattkus-Verzeichnis', ['The Stattkus-Verzeichnis (SV) is a catalogue of the musical compositions of the Italian composer Claudio Monteverdi.', ' The catalogue was published in 1985 by Manfred H. Stattkus (\"Claudio Monteverdi: Verzeichnis der erhaltenen Werke\").', ' A free, basic second edition of the catalogue is available online.']], ['Monteverdi (crater)', ['Monteverdi is a crater on Mercury.', ' It has a diameter of 138 kilometers.', ' Its name was adopted by the International Astronomical Union in 1979.', ' Monteverdi is named for the Italian composer Claudio Monteverdi, who lived from 1567 to 1643.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.755\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a79240955429974737f79a4', 'answer': 'Basset Bleu de Gascogne', 'question': 'Which dog is native to France, the Basset Bleu de Gascogne or the Bulldog?', 'supporting_facts': [['Basset Bleu de Gascogne', 0], ['Basset Bleu de Gascogne', 3], ['Bulldog', 0]], 'context': [['Landes de Gascogne Regional Natural Park', ['Landes de Gascogne Regional Natural Park (French: \"Parc naturel régional des Landes de Gascogne\") is a protected area of pine forest, wetland and oceanic coastline located in the Aquitaine region of southwestern France.']], ['Griffon Bleu de Gascogne', ['The Griffon Bleu de Gascogne (FCI No.32) Is a breed of dog of the scenthound type, originating in France, and is a versatile hunting dog, used on small and large game, in packs or individually.', ' The Griffon Bleu de Gascogne has a speckled, rough coat.']], ['Ariegeois', ['The Ariegeois is a breed of dog from the \"département\" of Ariège in the Midi-Pyrenées region of southern France.', ' It is a medium-sized pack-hunting scenthound deriving from crossing of Grand Bleu de Gascogne and Grand Gascon-Saintongeois hounds with local Briquet dogs.', ' It is used both as a courser and for driving game to waiting guns.', ' While most successful with hares, it is also used for hunting deer and boar.', ' It is distinguished by its friendly nature with other hounds and affection for human companions.']], ['Bulldog', ['The Bulldog is a medium-sized breed of dog commonly referred to as the English Bulldog or British Bulldog.', ' Other scent-hound breeds include the Small Greek Domestic Dog, Irish Wolfhound, Bluetick Coonhound, Finnish Lapphund, and the Basset Hound.', ' The Bulldog is a muscular, hefty dog with a wrinkled face and a distinctive pushed-in nose.', ' The American Kennel Club (AKC), The Kennel Club (UK), and the United Kennel Club (UKC) oversee breeding records.', ' Bulldogs were the fourth most popular purebreed in the US in 2007 according to the American Kennel Club.']], ['Petit Bleu de Gascogne', ['The Petit Bleu de Gascogne (FCI No.31) Is a breed of dog of the scenthound type, originating in France and used for hunting in packs.', \" Today's breed is the descendant of a very old type of large hunting dog.\", ' The Petit Bleu de Gascogne is not a small (petite) dog, the name comes from its use on small game.']], ['Côtes de Gascogne', ['Côtes de Gascogne is a wine-growing district in Gascony producing principally white wine.', ' It is mainly located in the département of the Gers in the French region Midi-Pyrénées, and it belongs to the wine region South West France.', ' The designation Côtes de Gascogne is used for a \"Vin de Pays\" (\"country wine\") produced in the Armagnac area.', ' The decree of 13 September 1968 created the difference between a \"Vin de Pays\" and simpler table wine, the so-called \"Vin de table\".', ' The designation \"Côtes de Gascogne\" obliges the producers to respect the stricter rules and production standards, which were adopted with the decree of 25 January 1982.']], ['Floc de Gascogne', ['The Floc de Gascogne is a regional apéritif from the Côtes de Gascogne and Armagnac regions of Sud-Ouest wine region of France.', ' It is a \"vin de liqueur\" fortified with armagnac, the local brandy.', ' It has had \"Appellation d\\'origine contrôlée\" status since 1990.']], ['Gros Manseng', ['Gros Manseng (sometimes translated: Large Manseng, rarely \"Big Manseng\") is a white wine grape variety that is grown primarily in South West France, and is part of the Manseng family.', ' It produces dry wines in the Jurançon and Béarn regions of Southwest France.', ' In Gascony it is permitted in the Pacherenc du Vic-Bilh \"Appellation d\\'origine contrôlée\" (AOC), in the Côtes de Gascogne and in the Floc de Gascogne.']], ['Basset Bleu de Gascogne', ['The Basset Bleu de Gascogne (] ), also known as the Blue Gascony Basset, is a long-backed, short legged breed of dog of the hound type.', ' The breed originated in the Middle Ages, descended from the Grand Bleu de Gascogne.', ' It nearly became extinct around the early 19th century; its salvation was attributed to one Alain Bourbon.', ' A French native breed, it is rare outside of its homeland.', ' It is recognized internationally by the Fédération Cynologique Internationale, in the UK by The Kennel Club, and by the United Kennel Club in the United States.', ' The \"bleu\" of its name is a reference to its coat which has a ticked appearance.']], ['Grand Bleu de Gascogne', ['The Grand Bleu de Gascogne (FCI No.22) is a breed of dog of the scenthound type, originating in France and used for hunting in packs.', \" Today's breed is the descendant of a very old type of large hunting dog, and is an important breed in the ancestry of many other hounds.\"]]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.756\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae530e355429908b6326561', 'answer': \"What's My Line\", 'question': '\"Twenty Two\" is an episode of the series \"The Twilight Zone.\" adapted from an anecdote by a publisher who made appearance in what panel game show?', 'supporting_facts': [['Twenty Two (The Twilight Zone)', 0], ['Twenty Two (The Twilight Zone)', 1], ['Bennett Cerf', 0], ['Bennett Cerf', 1]], 'context': [['In His Image', ['\"In His Image\" is an episode of the American television anthology series \"The Twilight Zone\" aired on January 3, 1963.', ' This was the first episode of the fourth season.', ' Each episode was expanded to an hour (with commercials) from \"In His Image\" until \"The Bard\\'\".', ' The fourth season is the only season of \"The Twilight Zone\" to have each episode one hour long.', ' In this episode, a man finds his hometown is suddenly inconsistent with his memories of it and begins experiencing irrational urges to commit murder, two mysteries which together lead him to an unpleasant discovery about his identity.']], ['Twilight Zone (Golden Earring song)', ['\"Twilight Zone\" is a 1982 hit by Dutch band Golden Earring.', ' It was written by the band\\'s guitarist George Kooymans, who got the inspiration from a book by Robert Ludlum, \"The Bourne Identity\".', ' \"Twilight Zone\" appears on their 1982 album \"Cut\" and pays tribute to the 1960s television series \"The Twilight Zone\".', ' It spent more than half a year (27 weeks) on the U.S. Pop charts.', ' It was the group\\'s sole Top 10 Pop single on the US \"Billboard\" Hot 100 and hit No. 1 on the \"Billboard\" Top Album Tracks chart, the band\\'s only No. 1 hit in America.']], ['The Twilight Zone (Rush song)', ['\"The Twilight Zone\" is the third track on Rush\\'s album \"2112\".', ' It was the last track written and recorded for the album.', ' It was the first single to be released from \"2112\".', ' As with most Rush songs, the lyrics are written by Neil Peart, and the music by Geddy Lee and Alex Lifeson.', ' It is based on two episodes of \"The Twilight Zone\": \"Will the Real Martian Please Stand Up?', '\" (first verse) and \"Stopover in a Quiet Town\" (second verse).', ' Rush had dedicated their previous album, \"Caress of Steel\", to the memory of \"The Twilight Zone\" creator Rod Serling.']], ['Twenty Two (The Twilight Zone)', ['\"Twenty Two\" is episode 53 of the American television series \"The Twilight Zone.\"', ' The story was adapted by Rod Serling from a short anecdote in the 1944 Bennett Cerf Random House anthology \"Famous Ghost Stories,\" which itself was an adaptation of \"The Bus-Conductor,\" a short story by E. F. Benson published in \"The Pall Mall Magazine\" in 1906.']], ['Time Enough at Last', ['\"Time Enough at Last\" is the eighth episode of the American television anthology series \"The Twilight Zone\".', ' The episode was adapted from a short story written by Lynn Venable (pen name of Marilyn Venable).', ' The short story appeared in the January 1953 edition of the science fiction magazine \"If: Worlds of Science Fiction\" about seven years before the television episode first aired.', ' \"Time Enough at Last\" became one of the most famous episodes of the original \"Twilight Zone\" and has been frequently parodied since.', ' It is \"the story of a man who seeks salvation in the rubble of a ruined world\" and tells of Henry Bemis , played by Burgess Meredith, who loves books, yet is surrounded by those who would prevent him from reading them.', ' The episode follows Bemis through the post apocalyptic world, touching on such social issues as anti-intellectualism, the dangers of reliance upon technology, and the difference between aloneness (solitude) and loneliness.']], ['Bennett Cerf', ['Bennett Alfred Cerf (May 25, 1898 – August 27, 1971) was an American publisher, one of the founders of American publishing firm Random House.', ' Cerf was also known for his own compilations of jokes and puns, for regular personal appearances lecturing across the United States, and for his television appearances in the panel game show \"What\\'s My Line?', '\"']], ['Many, Many Monkeys', ['\"Many, Many Monkeys\" is an episode of the television series \"The Twilight Zone\", first broadcast in 1989.', \" The episode was written in 1964 for the final season of the show's original black-and-white run by producer William Froug, but though CBS bought the script, they chose not to use it.\", ' Froug believed that they found it \"too grotesque.\"', ' It remained shelved for more than twenty years until it was made as an episode in the third and final season of the 1980s \"Twilight Zone\" revival.']], ['Panel show', ['A panel show or panel game is a radio or television game show in which a panel of celebrities participates.', ' Participants may compete with each other, such as on \"The News Quiz\"; facilitate play by non-celebrity contestants, such as on \"Match Game\"/\"Blankety Blank\"; or do both, such as on \"Wait Wait Don\\'t Tell Me\".', ' The genre can be traced to 1938, when \"Information Please\" debuted on U.S. radio.', ' The earliest known television panel show is \"Play the Game\", a charades show in 1946.', ' The modern trend of comedy panel shows can find early roots with \"Stop Me If You\\'ve Heard This One\" in 1939 and \"Can You Top This?', '\" in 1940.', ' While panel shows were more popular in the past in the U.S., they are still very common in the United Kingdom.']], ['The Twilight Zone Tower of Terror', ['\"The Twilight Zone\" Tower of Terror, also known as Tower of Terror, is an accelerated drop tower dark ride located at Disney\\'s Hollywood Studios, Tokyo DisneySea, Walt Disney Studios Park, and formerly located at Disney California Adventure Park.', ' Except for the Tokyo DisneySea version, the attractions are inspired by Rod Serling\\'s anthology television series, \"The Twilight Zone\", and take place in the fictional Hollywood Tower Hotel in Hollywood, California.', ' The Tokyo version, which features an original story line not related to \"The Twilight Zone\", takes place in the fictional Hotel Hightower.', ' All three versions place riders in a seemingly ordinary hotel elevator, and present the riders with a fictional backstory in which people have mysteriously disappeared from the elevator under the influence of some supernatural element many years prior.']], ['Night Gallery', ['Night Gallery is an American anthology series that aired on NBC from 1969 to 1973, featuring stories of horror and the macabre.', ' Rod Serling, who had gained fame from an earlier series, \"The Twilight Zone\", served both as the on-air host of \"Night Gallery\" and as a major contributor of scripts, although he did not have the same control of content and tone as he had on \"The Twilight Zone\".', ' Serling viewed \"Night Gallery\" as a logical extension of \"The Twilight Zone\", but while both series shared an interest in thought-provoking dark fantasy, more of \"Zone\"\\' s offerings were science fiction while \"Night Gallery\" focused on horrors of the supernatural.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.756\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae731ae5542991bbc9761c1', 'answer': 'Halliwell Jones Stadium', 'question': 'Where do the team that John Grant joined after the 1972 World Cup now play their games ?', 'supporting_facts': [['John Grant (rugby league)', 3], ['Warrington Wolves', 1]], 'context': [['Gary Sullivan (rugby league)', ['After playing for Kurri Kurri, Sullivan moved to Sydney to play in the New South Wales Rugby Football League premiership for Newtown in 1970.', \" After only six first grade games he was selected for Australia's victorious 1970 World Cup squad at lock.\", \" Following Ron Coote's decision to stand down from representative football in 1972, Sullivan made his New South Wales début before being selected in two Tests against New Zealand.\", \" He scored two tries in Australia's 36-11 win at the SCG before playing in the Second Test in Brisbane.\", \" At the end of the year Sullivan was selected for Australia's 1972 World Cup campaign.\", ' He was selected to play at lock forward in the tournament final against Great Britain which was drawn at 10-10.']], ['World Cup Live', ['World Cup Live is a soccer related news and analysis program.', ' It airs on ESPN, ESPN2 and ABC every four years during the FIFA World Cup.', ' Lead commentators and specialists dissect worldwide matches both through a live feed, and after they have already been played.', ' \"World Cup Live\" was created for the 2006 World Cup and continued through the 2010 World Cup.', ' It is planned to broadcast the upcoming 2014 FIFA World Cup as ESPN has English-language rights.', ' As for anchors, \"SportsCenter\" hosts Dave Revsine and Rece Davis worked with ESPN while Brent Musburger hosted the ABC airings.', ' Advertisements are not shown seeing as that play does not stop for two forty-five-minute half’s, other than a halftime report, during which, commercials are aired.', ' Logos are shown on the screen during broadcasting throughout the game and advertisements from sponsors can be seen before and after the game.', ' Both pregame and post-game segments are included with a large amount of games if time between matches permits, and are always aired for USMNT games.']], ['John Wilson (New Zealand rugby league)', ['John Wilson is a New Zealand rugby league player who represented his country in the 1972 World Cup.']], [\"Neo Geo Cup '98: The Road to the Victory\", [\"Neo Geo Cup '98: The Road to the Victory is a soccer video game based on the FIFA World Cup 1998, despite being released after the 1998 FIFA World Cup.\", \" It features 73 teams' countries.\", ' Each team enters a \"Regional Qualifying Round Final\" where it plays a team it actually played in the 1998 FIFA World Cup qualification.', ' For example: Spain would face Yugoslavia, an opponent it actually faced in its qualifying group.', ' Or Italy would face Russia, an opponent Italy faced in the UEFA play-offs.', ' If the player beats the opponent, it goes to a group much like the real life World Cup.', ' In fact, the team faces opponents that were actually in its group.', ' For example: Mexico would face the Netherlands, Belgium and South Korea.', ' It is a re-make of \"Super Sidekicks 3\".', ' However, animations and designs were exactly the same.', ' The only difference is teams to reflect the World Cup, kits again to reflect the World Cup, and players to resemble squads from the World Cup (teams that did not qualify use line-ups from friendly games and qualifiers).', ' Its slogan is \"We got the kick\".']], ['George Nicholls (rugby league)', ['George Nicholls was born on 14 May 1944 in Widnes, Lancashire.', ' He played for Rugby Football League club Widnes at prop forward in their 8-15 loss to Wigan in the 1971 Lancashire County Cup Final during the 1970–71 season at Knowsley Road, St. Helens on Saturday 28 August 1971.', ' While playing club football for Widnes, Nicholls played at Loose forward for the Great Britain Lions who retained the 1972 World Cup.']], ['List of dual Rugby World Cup winners', ['Since the inception of the Rugby World Cup in 1987, a total of twenty rugby players have been dual Rugby World Cup winners.', ' The exclusive club initially included five Australian players, John Eales, Phil Kearns, Dan Crowley, Jason Little and Tim Horan, who were part of both the 1991 Rugby World Cup and 1999 Rugby World Cup Wallabies squads.', ' They were joined in 2007 by South African player Os Du Randt, who played for the Springboks in their 1995 Rugby World Cup and 2007 Rugby World Cup victories.', \" They were joined by 14 All Blacks in 2015, who played in New Zealand's 2011 Rugby World Cup and 2015 Rugby World Cup victories.\", \" Of these twenty, New Zealand's Sonny Bill Williams, Jerome Kaino and Sam Whitelock have played in a record fourteen consecutive World Cup wins, while Richie McCaw is the first player to captain his nation to two titles.\"]], ['Geoff Starling', ['In 1970 Starling was playing in the Jersey Flegg competition for the Balmain club.', ' The following season he started playing first grade, gaining selection for the Australian national team, becoming Kangaroo No. 459, and the youngest player to ever represent Australia.', ' He was 18 years and 181 days old when playing a tour match against a New Zealand XIII at Huntly.', ' That season he also played for the New South Wales side.', ' The following year he made his Test match début against New Zealand.', ' Starling was also selected to represent Australia in the 1972 World Cup, playing in the final which was drawn with Great Britain.', ' In 1973 Starling was selected to go on the end of season Kangaroo tour, helping Australia to victory in the Ashes series.']], ['John Grant (rugby league)', ['John Grant (born 19 March 1950) is an Australian businessman, rugby league football administrator and current chairman of the Australian Rugby League Commission which controls rugby league in Australia.', ' A former player of the 1970s, he was a Queensland interstate representative three-quarter back and a member of the Australian team which lost the 1972 World Cup to Great Britain in France.', \" Grant had been playing his club football for the Brisbane Rugby League's Souths club under Wayne Bennett.\", ' Following the World Cup, Grant joined English club Warrington, playing for them during their table-topping 1972–73 season.']], ['Roy Christian', ['Born in Auckland to Norfolk Island parents, Roy Christian is a direct descendant of Fletcher Christian, a figure in the 1789 Mutiny on the Bounty.', ' Christian played his first Test match in 1965 against Australia.', ' While playing for Otahuhu in 1966, Christian was awarded the Lipscombe Cup for Premier One sportsman of the year.', ' However, injury caused him to miss the 1968 World Cup.', ' Christian was part of the Auckland side that defeated Australia in 1969.', \" He was appointed captain of the New Zealand national side in 1970 and played in that year's World Cup.\", ' In 1971 Christian captained New Zealand to a famous victory against Australia at Carlaw Park.', ' Also in 1971, his Kiwis side became the first New Zealand touring team to win a test series in Britain.', ' Christian was awarded a MBE in the 1972 Birthday Honours \"for services to rugby league football\".', ' The 1972 World Cup was the last time Christian represented New Zealand.', ' He retired with little fanfare as no test matches were scheduled for 1973.', ' He had played in 74 matches for the Kiwis, including 32 tests.', ' After retirement Christian served as the Otahuhu Leopards chairman before becoming a Minister in the Presbyterian Church.', ' In 2007 he was inducted as one of the New Zealand Rugby League\\'s \"Legends of League\".']], ['Warrington Wolves', ['Warrington Wolves R.L.F.C. is a professional rugby league football club based in Warrington, England that competes in Super League.', ' They play at the Halliwell Jones Stadium, having moved there from Wilderspool in 2003.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.757\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a716f575542994082a3e82f', 'answer': 'Joseph John Campbell', 'question': 'Was it  Ishmael Reed or Joseph Campbell who said Follow your bliss?', 'supporting_facts': [['Joseph Campbell', 0], ['Ishmael Reed', 0]], 'context': [['The Signifying Monkey', ['The Signifying Monkey: A Theory of African-American Literary Criticism is a work of literary criticism and theory by American scholar Henry Louis Gates Jr. first published in 1988.', ' The book traces the folkloric origins of the African-American cultural practice of “signifying” and uses the concept of Signifyin(g) to analyze the interplay between texts of prominent African-American writers, specifically Richard Wright, Ralph Ellison, Zora Neale Hurston and Ishmael Reed.']], ['A Gathering of the Tribes (Cultural Organization)', [\"A Gathering of the Tribes is a multi cultural interdisciplinary arts organization founded by Dr. Steve Cannon at his home at 285 East third street in New York City's East Village.\", ' The Organization publishes a literary magazine and has now presented its fourteenth issue.', ' The collective also hosted a gallery and performance space where numerous exhibitions and concerts took place with notable artists and musicians such as the Sun Ra Arkestra, Chavisa Woods, Katherine Arnoldi, John Farris, Susan Scutti, Bob Holman, Ishmael Reed, Billy Bang and David Hammons.']], ['Joseph Campbell', ['Joseph John Campbell (March 26, 1904 – October 30, 1987) was an American mythologist, writer, and lecturer, best known for his work in comparative mythology and comparative religion.', ' His work covers many aspects of the human experience.', ' Campbell\\'s \"magnum opus\" is his book \"The Hero with a Thousand Faces\" (1949), in which he discusses his theory of the journey of the archetypal hero found in world mythologies.', \" Since the book's publication, Campbell's theory has been consciously applied by a wide variety of modern writers and artists.\", ' His philosophy has been summarized by his own often repeated phrase: \"Follow your bliss.\"']], ['Mumbo Jumbo (novel)', ['Mumbo Jumbo is a 1972 novel by African-American author Ishmael Reed.', ' Literary critic Harold Bloom cited the novel as one of the 500 most important books in the Western canon.', ' \"Mumbo Jumbo\" has remained in print for 45 years, since its first edition, and has been published in French, Italian, Spanish, Japanese, and British editions, with a Chinese translation currently in production.']], ['Before Columbus Foundation', ['The Before Columbus Foundation is a nonprofit organization founded in 1976 by Ishmael Reed, \"dedicated to the promotion and dissemination of contemporary American multicultural literature\".', ' The Foundation makes annual awards for books published in the US during the previous year that make contributions to American multicultural literature.']], ['Ishmael Reed', ['Ishmael Scott Reed (born February 22, 1938) is an American poet, novelist, essayist, songwriter, playwright, editor and publisher, who is known for his satirical works challenging American political culture.']], ['PEN Oakland', ['PEN Oakland is a branch of PEN, an international literary and human rights organization.', ' PEN Oakland was founded in 1989 by Ishmael Reed and co-founders Floyd Salas, Claire Ortalda and Reginald Lockett.', ' PEN Oakland annually sponsors the PEN Oakland/Josephine Miles Literary Award, named for the late poet and faculty member of U.C. Berkeley’s English Department.', ' The award honors well-known and emerging Bay Area and international authors for excellence in multicultural literature.', ' Past and present PEN Oakland board members include: Ishmael Reed, Floyd Salas, Robert Mailer Anderson, Tony R. Rodriguez, Lucha Corpi, John Curl, Elmaz Abinader, Al Young, Jack Foley (poet), Kim Addonizio, Opal Palmer Adisa, Gerald Nicosia, Ntozake Shange and Gary Soto.', ' PEN Oakland is based in Oakland, CA.']], ['The Last Days of Louisiana Red', ['The Last Days of Louisiana Red (1974) is a novel written by Ishmael Reed.', ' It is considered a model novel of the Black Arts Movement and contains many elements of postmodernism.', ' It continues the story of the character Papa LaBas introduced in Reed\\'s previous novel, 1972\\'s \"Mumbo Jumbo\".', ' The book revolves heavily around voodoo.']], ['Yellow Back Radio Broke-Down', ['Yellow Back Radio Broke-Down, by the African-American writer Ishmael Reed, is a satirical take on the traditional Western.', ' It is Ishmael Reed\\'s second novel, following \"The Freelance Pallbearers\" (1967), and was first published in 1969.', ' It tells the story of the Loop Garoo Kid, an African-American cowboy who practices the religion of Neohoodooism, and describes his struggle against established religion and cultural oppression.']], ['Sacred Ground (David Murray album)', ['Sacred Ground is an album by David Murray released on the Justin Time label.', ' Recorded in 2006 and released in 2007 the album features performances by Murray, Lafayette Gilchrist, Ray Drummond, and Andrew Cyrille which were composed for the soundtrack for Marco Williams\\' film \"Banished\" (2007) on American counties in the South and Midwest that expelled blacks between Reconstruction and the Great Depression.', ' The album features Cassandra Wilson on two tracks singing lyrics composed by Ishmael Reed.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.758\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae1fa2b5542997f29b3c1df', 'answer': 'Eminem', 'question': 'Who collaborated with Rihanna on the album release \"Numb\"?', 'supporting_facts': [['The Monster (song)', 2], ['Numb (Rihanna song)', 0]], 'context': [['Jarren Benton', ['Jarren Giovanni Benton (born October 26, 1981) is an American rapper from Decatur, Georgia.', ' In early 2012, he signed to rapper Hopsin\\'s independent record label Funk Volume and released a mixtape called \"Freebasing with Kevin Bacon\" in June 2012.', ' A year later, on June 11, 2013 he released his debut studio album \"My Grandma\\'s Basement\", which received positive critical reviews and debuted at number 152 on the \"Billboard\" 200.', ' On January 4, 2016, Jarren Benton posted a prank on Instagram, saying that he dropped his current label, \"Funk Volume,\" for a label no longer in existence entitled \"No Limits.\"', ' Fans and news outlets alike took the prank seriously, and spread the joke as truth on the internet because they refused to find solid evidence and verify confirmation of fact to the joke.', ' Both Funk Volume and Jarren Benton have disproved truth to the prank in the same night.', ' However, it has been officially confirmed the Funk Volume label has split up.', ' After the Funk Volume split up, Jarren has created his own record label under the name Benton Enterprises, choosing to go in his own direction to get his own brand out to the public, and to release his new album Slow Motion Vol.', ' 2.', ' The album originally was scheduled to be released on July 15th, but Jarren posted on social media on July 3rd, 2016 \"Due to technical difficulties, we will be releasing Slow Motion Vol.', ' 2 on July 22nd.\"', ' The album will be released on his new website and will be his first album release under his label Benton Enterprises, his first album release since the Funk Volume departure, and his first album released on his new website.']], ['Rob Dougan', ['Robert Don Hunter \"Rob\" Dougan ( ; born 1969) is an Australian composer, known for his genre-blending music.', ' Mixing elements of orchestral music, trip hop, and bluesy vocals, his work is tangentially relatable to electronic music.', ' He is known primarily for his breakthrough 1995 single \"Clubbed to Death (Kurayamino Variation)\", further popularised by 1999\\'s \"The Matrix\" soundtrack.', ' \"Clubbed to Death\" was re-released on his debut album \"Furious Angels\" in 2002, seven years after its initial release.', ' As well as providing several Clubbed To Death Variations most notably Kurayamino Variation he has also provided a Clubbed To Death Variation of the Moby classic Porcelain.', ' In 1995 he teamed up with Rollo to remix the U2 classic \"Numb\"; the remix was titled \"Numb (Gimme Some More Dignity Mix).\"']], ['The Libertine (album)', ['The Libertine: Music for the Film by Laurence Dunmore is the album release of Michael Nyman\\'s score for the 2004 film \"The Libertine\" directed by Laurence Dunmore.', \" It is the third release on Nyman's own label, MN Records, and the first to receive distribution in the United States, by Inner Knot Records.\", ' It is his 50th album release overall.', ' When Naxos Records began distributing MN Records in the United States in 2008, it was included and began appearing in large quantitites in stores.', \" This is Nyman's last score for a major motion picture to date, and his last soundtrack release, other than compilation soundtracks.\"]], ['Mack 10 discography', ['The discography of West Coast hip hop artist Mack 10 consists of eight studio albums, two compilation albums, twenty-two singles, and fifteen music videos.', ' He has also collaborated on two albums and was featured in two soundtrack albums.', ' After signing to Priority Records in 1995, Mack 10 released his self-titled debut album in June.', ' The album, produced by fellow rapper Ice Cube, saw considerable commercial success and went Gold in the US.', ' His prosperity continued when he released \"Based on a True Story\", which peaked at number fourteen on the US \"Billboard\" 200.', ' The rapper collaborated with Tha Dogg Pound to record \"Nothin\\' But the Cavi Hit\" which was released on the \"Rhyme & Reason\" soundtrack.', ' Mack 10\\'s 1998 release, \"The Recipe\", was the rapper\\'s third and final album to be certified Gold in the US by RIAA.', ' Mack 10\\'s album sales began to decline after his first compilation album release, \"Hoo-Bangin\\': The Mix Tape, Vol.', ' 1\".', ' His fourth studio album, \"The Paper Route\" (2000), debuted at number nineteen on the \"Billboard\" 200; however, it failed to earn the rapper any RIAA certifications.']], ['Numb (Rihanna song)', ['\"Numb\" is a song by Barbadian singer Rihanna from her seventh studio album \"Unapologetic\" (2012).', ' It features guest vocals by American rapper Eminem, making it the pair\\'s third collaboration since the two official versions of \"Love the Way You Lie\".', ' Following the album\\'s release, \"Numb\" charted on multiple charts worldwide including in Canada, the United Kingdom and the United States.']], ['The Monster (song)', ['\"The Monster\" is a song by American rapper Eminem, featuring guest vocals from Barbadian singer Rihanna, taken from Eminem\\'s album \"The Marshall Mathers LP 2\" (2013).', ' The song was written by Eminem, Jon Bellion, and Bebe Rexha, with production handled by Frequency.', ' \"The Monster\" marks the fourth collaboration between Eminem and Rihanna, following \"Love the Way You Lie\", its sequel \"Love the Way You Lie (Part II)\" (2010), and \"Numb\" (2012).', ' \"The Monster\" was released on October 29, 2013, as the fourth single from the album.', \" The song's lyrics present Rihanna coming to grips with her inner demons, while Eminem ponders the negative effects of his fame.\"]], ['Lemonjelly.ky', ['Lemonjelly.ky is the debut album release by downtempo/trip hop act Lemon Jelly.', ' Released on 23 October 2000, it is a compilation release, as all nine tracks from this album originated on the duo\\'s first three limited edition EPs: \"The Bath\", \"The Yellow\" and \"The Midnight\", although minor changes were made for the album release.']], ['To the Metal!', ['To The Metal!', ' is the tenth full-length studio album by Gamma Ray.', ' It was released on 29 January 2010.', ' To promote the album, the band did a tour with Freedom Call and Secret Sphere.', \" The album was recorded in Kai's own studio in Hamburg in autumn 2009.\", ' The band has recorded 12 songs.', ' Ten of them were featured on the regular album release, while the other two were bonus tracks appearing on the different editions of the album.', ' The band described some of the songs on the official website, revealing that there would be a full-throttle number called \"Rise\", a rhythmic and melodically diverse song called \"Time To Live\", and a multilayered anthem titled \"All you need to know\", featuring ex-Helloween frontman Michael Kiske.', ' Other songs mentioned are \"No Need to Cry\", a song written by Dirk Schlächter about the death of his father, \"To The Metal\", a song they played on various festivals before the album release and the atmospheric and dense \"Empathy\".', ' It is the last album featuring drummer Dan Zimmermann.']], ['The Queen of Hardships', ['The Queen of Hardships (sometimes credited as Queen of Hardships) is the 13th studio album by cantopop singer Prudence Liew.', ' This album marks the first Cantonese language studio album release from Liew in 15 years since her 1994 release of \"夜有所思，日有所夢 Thoughts in the Night, Dreams During the Day\" and the first studio album release in 9 years.', ' Her previous studio album was the mandopop album, \"愛自己 Love Yourself\" released in Taiwan in 2000.']], ['Live at the El Rey (EP)', ['Live at the El Rey was a limited edition live album recorded by New Orleans electro-rock band Mutemath.', ' The album was recorded live at the El Rey Theatre in Los Angeles on the Album Release Tour in January 2006 and features a selection of six songs from the actual set performed on the Album Release Tour.', ' Video of the performances was also recorded and used for promotional materials on various online media outlets including AOL Music and was included in the UK physical release of the single \"Typical\".', ' Only 25,000 copies of the EP were printed and sold as part of an exclusive limited edition version of the group\\'s self-titled debut album \"Mutemath\" when it was re-released in the US on Warner Bros.', ' Records on September 26, 2006.', ' International releases also include the EP, but the number of copies printed is unknown.', ' It is also available on iTunes as bonus tracks for the \"deluxe\" version of the self-titled album \"MuteMath\"']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.759\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ade0fbb5542995b365fabfb', 'answer': 'cat', 'question': 'No Smoking is a film loosely based upon the short story Quitters, Inc., which of three stories that are connected by the presence of what animal? ', 'supporting_facts': [['No Smoking (2007 film)', 0], ['No Smoking (2007 film)', 2], [\"Cat's Eye (1985 film)\", 0], [\"Cat's Eye (1985 film)\", 1], [\"Cat's Eye (1985 film)\", 3]], 'context': [['The 3 Worlds of Gulliver', ['The 3 Worlds of Gulliver is a 1960 Eastman Color Columbia Pictures fantasy film loosely based upon the 18th-century Irish novel \"Gulliver\\'s Travels\" by Jonathan Swift.', ' The film stars Kerwin Mathews as the title character, June Thorburn as his fiancée Elizabeth, and child actor Sherry Alberoni as Glumdalclitch.']], ['No Smoking (2007 film)', ['No Smoking is a 2007 Indian neo-noir psychological thriller film written and directed by Anurag Kashyap and co-produced by Vishal Bhardwaj and Kumar Mangat.', ' The film stars John Abraham, Ayesha Takia, Ranvir Shorey and Paresh Rawal in the lead roles, while Bipasha Basu appears in an Item number.', ' The film is loosely based upon the short story \"Quitters, Inc.\" by Stephen King, which was previously adapted as one of three segments featured in Hollywood anthology film, \"Cat\\'s Eye\" (1985).', ' It became the first Indian film to be adapted from a Stephen King short story.', ' The story follows K (Abraham) a self-obsessed, narcissist chain smoker who agrees to kick his habit to save his marriage and visits a rehabilitation centre, but is caught in a labyrinth game by Baba Bengali (Rawal), the man who guarantees he will make him quit.']], ['The Adjustment Bureau', ['The Adjustment Bureau is a 2011 American science fiction thriller film loosely based on the Philip K. Dick short story, \"Adjustment Team\".', ' The film was written and directed by George Nolfi, produced by Chris Moore and stars Matt Damon and Emily Blunt.', ' The cast also includes Anthony Mackie, John Slattery, Michael Kelly, and Terence Stamp.', ' The film tells the story of a young man who discovers that what appear to be chance events in his life are controlled by a technologically advanced intelligence network.', ' After an event not planned by these controllers occurs – a romantic encounter with a young dancer – he struggles against their manipulation despite their promise of a great future for him.']], ['Teen Kanya', ['Teen Kanya is a 1961 Indian Bengali anthology film directed by Satyajit Ray, and based upon short stories by Rabindranath Tagore.', ' The title means \"Three Girls\", and the film\\'s original Indian release contained three stories.', ' However, the international release of the film contained only two stories, missing out the second (\"Monihara: The Lost Jewels\").', ' This version was released on VHS in 1997 under the title \"Two Daughters\".', ' However, there are now DVD versions available that contain all three films.', ' The Academy Film Archive preserved the international version of \"Teen Kanya\" in 1996.']], [\"Cat's Eye (1985 film)\", ['Cat\\'s Eye (also known as \"Stephen King\\'s Cat\\'s Eye\") is a 1985 American anthology horror film directed by Lewis Teague and written by Stephen King.', ' It comprises three stories, \"Quitters, Inc.\", \"The Ledge\", and \"General\".', ' The first two are adaptations of short stories in King\\'s \"Night Shift\" collection, and the third is unique to the film.', ' The three stories are connected only by the presence of a traveling cat, which plays an incidental role in the first two and is a major character of the third.']], ['Calling You (short story collection)', ['Calling You (Japanese: きみにしか聞こえない , Hepburn: Kimi ni Shika Kikoenai ) is a Japanese fictional short story collection written by Otsuichi and published on May 31, 2001 by Kadokawa Shoten.', ' All three stories in \"Calling You\" are stories focused on unusual friendships with a supernatural twist.', ' In December 2003, a manga adaptation written and illustrated by Setsuri Tsuzuki was published by Kadokawa.', ' The \"Calling You\" manga only includes the first two stories of the novel, and makes some changes to both of those stories.', ' Both the novel and manga adaptations were given an English language release in North America by Tokyopop.']], ['Gariyoshi', ['Gariyoshi (গৰীয়সী) is an Assamese language monthly literary magazine published by the Sahitya-Prakash, Tribune Building, Guwahati.', ' It was founded by Chandra Prasad Saikia, who was also the first editor.', ' The magazine is published monthly.', ' Current editor Dr. Lakshmi Nandan Bora assumed the post in April 2009.', ' Bora was preceded by Harekrishna Deka.', ' \"Goriyoshi\" is instrumental in nurturing and projecting several talented short story writers and poets including Dhanada Debi, Jayanta Kumar Chakraborty, Arnab Jan Deka, Manikuntala Bhattacharya, Birinchi Kumar Rabha, Jiban Narah, Neelim Kumar and others.', ' The magazine also collaborated with Katha International Short Story Festival in 2004 in creating All-India Katha-Goriyoshi Awards for best Assamese short stories.', ' Dhrubajyoti Sarma, Arnab Jan Deka and Ratna Bharali Talukdar had been the recipients of those awards, whose short stories had been translated into English and read over in presence of an international galaxy of story writers and literary critics at Katha International Short Story Festival 2004.']], ['The Last Question', ['\"The Last Question\" is a science fiction short story by American writer Isaac Asimov.', ' It first appeared in the November 1956 issue of \"Science Fiction Quarterly\" and was anthologized in the collections \"Nine Tomorrows\" (1959), \"The Best of Isaac Asimov\" (1973), \"Robot Dreams\" (1986), the retrospective \"Opus 100\" (1969), and in \"Isaac Asimov: The Complete Stories, Vol.', ' 1\" (1990).', \" It was Asimov's favorite short story of his own authorship, and is one of a loosely connected series of stories concerning a fictional computer called Multivac.\", ' The story overlaps science fiction, theology, and philosophy.']], ['Vampires vs. Zombies', ['Vampires vs. Zombies is an independent horror film loosely based upon J. Sheridan Le Fanu\\'s classic 1872 novel \"Carmilla\".', \" Unlike Le Fanu's story, however, most of the action in the film takes place inside a car.\", ' The title and the cover were obviously inspired by the horror film \"Freddy vs. Jason\", it\\'s unclear if it was intended as a mockbuster of that film or not.']], ['Time Pussy', ['\"Time Pussy\" is an early science fiction short story by American writer Isaac Asimov.', ' It was the third of three stories Asimov wrote for John W. Campbell for a new category of science fiction tall tales in \"Astounding Science Fiction\" called \"Probability Zero\".', ' Campbell rejected the first two stories, \"Big Game\" and \"First Law\", since they were not what he was looking for, but he accepted \"Time Pussy\", albeit unenthusiastically.', ' Campbell also wanted to run the story under a pseudonym, since he wanted to encourage new writers to write \"Probability Zero\" stories.', ' Asimov agreed, and chose the name George E. Dale at random.', ' The story appeared pseudonymously in the April 1942 issue of \"Astounding\" and was reprinted under Asimov\\'s name in the 1972 collection \"The Early Asimov\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.759\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae76d185542997ec2727641', 'answer': 'jewelry designer', 'question': \"What is the occupation of the Miami Dolphins owner's wife?\", 'supporting_facts': [['Kara Ross', 0], ['Kara Ross', 1], ['Stephen M. Ross', 4]], 'context': [['Kara Ross', ['Kara Ross (née Gaffney; born 1966/1967) is an American jewelry designer.', ' She is the wife of American billionaire Stephen M. Ross.']], ['Steve Clark (American football)', ['Stephen Spence Clark (born August 2, 1960) is a former professional American football player who played [[defensive tackle]and offensive guard ] for five seasons for the [[Miami Dolphins]].', ' He also played on two state championship teams in high school which were a combined (25-1) over two years and was a five team all-American including \"Parade Magazine\", he was also named Most Valuable Player of the state of Utah.', ' At the University of Utah he was named two time All-WAC defensive tackle, Defensive Most Valuable Player of the Western Athletic Conference and First Team All-American.', ' He also played in the East-West Shrine Game and was named MVP of the Senior Bowl.', ' After the Senior Bowl he was drafted by Don Shula and The Miami Dolphins, his second year in the NFL he played both ways in a pre-season game and Coach Shula knew he had a guy that could back up every position on the offensive and defensive line as well as long snap.', ' He earned a starting position at right guard and played against [[William Perry (American football)|the Fridge]] when the Dolphins beat the Chicago Bears on Monday Night Football to help keep the undefeated Dolphin record intact.', ' In the NFL, he also played on two Super Bowl teams with the Miami Dolphins and was the starting right guard before being injured.', ' Just recently Steve was named to the top 100 greatest players in the history of the University of Utah actually being named 9th best of All-Time.']], ['International Champions Cup', ['The International Champions Cup (ICC) is an annual club association football exhibition competition.', ' It features club teams from Europe playing pre-season friendly matches, originally in the United States and Canada, but in the years since also in venues in China, Australia, Mexico and across Europe.', ' The ICC was founded by Relevent Sports, a division of RSE ventures based out of northern New Jersey, a sports venture firm founded in 2012 by billionaire real estate magnate and Miami Dolphins owner Stephen Ross and Matt Higgins, a former executive with the New York Jets.', ' It replaced the World Football Challenge, which had featured a more even distribution of European- and American-based sides.']], ['1979 Miami Dolphins season', ['The 1979 Miami Dolphins season was the 14th year of existence for the Miami Dolphins franchise.', ' Prior to the start of the season the Dolphins re-signed Larry Csonka who left to join the WFL after the 1974 season.', ' Despite struggles from Bob Griese all year, the Dolphins finished 10-6 and won their first division title in five years.', ' Among the season highlights were the Dolphins 19th and 20th consecutive wins over the Buffalo Bills.', ' For the entire decade of the 1970s (1970–79) the Dolphins hold a perfect 20-0 record over the Bills, which contributed to O.J. Simpson never seeing any postseason success in his career.', ' In the Divisional Playoff the Dolphins were no match for the Pittsburgh Steelers who jumped out to a 20-0 lead in the 1st Quarter to win 34-14 on their way to their second Straight Super Bowl title.']], ['Drone Racing League', ['The Drone Racing League (DRL) is a professional motorsport league where players race drones.', ' It was founded in 2015 by the Miami Dolphins owner Stephen Ross.']], ['Stephen M. Ross', ['Stephen M. Ross (born May 10, 1940) is an American real estate developer, philanthropist and sports team owner.', ' Ross is the chairman and majority owner of The Related Companies, a global real estate development firm he founded in 1972.', ' Related is best known for developing the Time Warner Center, where Ross lives and works, as well as the Hudson Yards Redevelopment Project.', ' According to \"Forbes\" magazine, Ross had a net worth of $12 billion.', ' Ross is also the principal owner of the Miami Dolphins and Hard Rock Stadium.']], ['1981 Miami Dolphins season', ['The 1981 Miami Dolphins season was the 16th year of existence for the Miami Dolphins franchise.', ' With the retirement of Bob Griese not much was expected out of the Dolphins.', ' The Dolphins Defense, which became known as the Killer Bees because of the number of players whose last name began with the letter B.', ' The Bees were Bill Barnett, Bob Baumhauer, Lyle Blackwood, Kim Bokamper, and Bob Brudzinski anchored a strong team.', ' They finished 11-4-1, as Don Shula reached a milestone by winning his 200th game of his coaching career.', ' In the Divisional Playoffs against the San Diego Chargers the Dolphins fell behind 24-0 early in front of a sold out crowd at the Orange Bowl.', ' With time running out in the first Half the Dolphins desperately needed a score to get back in the game.', ' Out of nowhere the Dolphins ran the old schoolyard hook and lateral play to success.', ' On the play Quarterback Don Strock threw a pass over the middle to WR Duriel Harris who lateraled to WR Tony Nathan who ran the ball in for Touchdown.', ' The play sparked the Dolphins who came back, and took a lead in the 4th Quarter.', ' However, the Killer Bees could not contain Chargers QB Dan Fouts who tied the game, and forced overtime where the Chargers won the game on a Rolf Bernershka Field Goal in the 14th minute of overtime.']], ['1982 Miami Dolphins season', [\"The 1982 Miami Dolphins season was the team's seventeenth in the National Football League.\", ' The team was coming off an unexpected 11-4-1 1981 season and a devastating loss to the San Diego Chargers in the Divisional Round the previous season in a game dubbed the Epic in Miami.', ' The Dolphins had clinched the 2 seed and were picked by many to reach the Super Bowl during the 1981 season.', ' Because of the high number of picks to reach the Super Bowl the previous season, many more fans picked them to win it during the 1982 season.', ' The Dolphins looked to improve on their 11-4-1 record from 1981.', \" However, a players strike cancelled 7 of the team's 16 games.\", ' Because of this, the NFL schedule was shrunk to 9 games.', ' The Dolphins started out fresh, winning their first 2 games prior to the strike.', ' When season play resumed 2 months later, the Dolphins defeated the Buffalo Bills 9-7 in Buffalo to clinch a 3-0 start.', ' After a loss to Tampa Bay, they defeated the Minnesota Vikings 22-14.', ' The next week, they lost a brisk game against the Patriots 3-0 in a game called the Snowplow Game.', ' The Dolphins would then win 3 straight games to end the season 7-2, tied for 2nd in the AFC with the Cincinnati Bengals.', ' The Dolphins won 2nd place over them by virtue of a series of tiebreakers.', ' In the playoffs, they defeated the Patriots in a rematch by the score of 28-13.', ' They then defeated the Chargers in a rematch of the 1981 Divisional Playoffs by a score of 34-13.', ' In the AFC Championship game, they shutout the Jets, 14-0 to reach the Super Bowl for the first time since 1973.', \" In Super Bowl XVII, they lost to the Redskins 27-17 in a rematch of Super Bowl VII which concluded Miami's perfect 1972 season.\"]], ['Shawn Wooden', ['Shawn Wooden (born October 23, 1973,) is a former American football safety who played in the National Football League for 9 seasons for the Miami Dolphins and the Chicago Bears.', ' Wooden was drafted in the 6th round by Jimmy Johnson, the then coach of the Miami Dolphins.', ' He played for the Dolphins for four seasons and then signed a free agent contract with the Chicago Bears in the 2000 football season.', ' After one year with the Chicago Bears, he returned to the Miami Dolphins for the remainder of his career.', ' He is currently a financial advisor with Wooden Wealth Strategies.']], ['List of Miami Dolphins broadcasters', [\"The Miami Dolphins' flagship radio station is AM 560 WQAM.\", ' WQAM has previously carried Dolphins broadcasts during the 1997-04, and 2007-09 NFL Seasons.', ' The radio broadcast team features Jimmy Cefalo providing play-by-play commentary and Joe Rose providing color commentary during preseason games, along with Griese for regular season games.', ' Griese replaced longtime color commentator Jim Mandich, who played for the Dolphins under Don Shula.', ' Mandich lost his fight with cancer in 2011, opening the door for Griese as his replacement.', ' The Miami Dolphins Radio Network is a statewide network of radio stations in Florida.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.761\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae61ac355429929b0807ac6', 'answer': 'Oscar Leonard Carl Pistorius', 'question': 'H Pistorius & Co is part of the reason for the wealth of the family that includes which South African sprinter?', 'supporting_facts': [['H Pistorius &amp; Co', 1], ['Oscar Pistorius', 0], ['Oscar Pistorius', 1]], 'context': [['Ncincihli Titi', ['Ncincilili Titi (born 15 December 1993) is a South African sprinter who competes primarily in the 200 metres events.', ' He finished fourth at the 2014 African Championships, as well as at the 2015 Summer Universiade.']], ['Tamzin Thomas', ['Tamzin Thomas (born 6 October 1997) is a South African sprinter.', ' She won two medals at the 2015 African Junior Championships.']], ['Tsholofelo Thipe', ['Tsholofelo Thipe (née Selemela) (born 9 December 1986) is a South African sprinter, who specialized in the 400 metres.', ' She set a personal best time of 51.15 seconds by winning the 400 metres event at the 2009 South African Championships in Stellenbosch.', ' She was born in Rustenburg, North West Province.']], ['Corné du Plessis', ['Corné du Plessis (born 20 March 1978) is a South African sprinter.', ' Together with Morne Nagel, Lee-Roy Newton and Mathew Quinn he won a silver medal in 4 x 100 metres relay at the 2001 World Championships in Athletics.', ' Their time of 38.47\\xa0seconds was a South African record.', ' Earlier in the season he won the bronze medal in the 200 metres at the 2001 Summer Universiade.']], ['Oscar Pistorius', ['Oscar Leonard Carl Pistorius OIB ( ; ] ; born 22 November 1986) is a South African sprint runner and convicted murderer.', \" Both of Pistorius' legs were amputated below the knee when he was 11 months old.\", ' He was the tenth athlete to compete at both the Paralympic Games and Olympic Games, competing in sprint events for below-knee amputees in Paralympic events, and in non-disabled sprint events.']], ['Paul Nash (athlete)', ['Paul Nash (born 1947) is a South African sprinter who tied the 100-metre world record four times in 1968 with a time of 10.0 seconds.', ' He attended Michaelhouse school in the province of KwaZulu-Natal, South Africa.']], ['Pieter Smith', ['Pieter Smith (born April 3, 1987 in Upington) is a South African sprinter, who specialized in the 400 metres.', ' He set his personal best time of 45.63 seconds by winning the 400 metres event at the 2009 South African Championships in Stellenbosch.']], ['Justine Palframan', ['Justine Palframan (born 4 November 1993) is a South African sprinter specialising in the 200 and 400 metres.', ' She won the 400 m event at the 2015 Summer Universiade.', ' She also represented South Africa at the IAAF 2013 World Championships and 2016 Olympics.']], ['H Pistorius &amp; Co', ['H Pistorius & Co is a private South African company based in Pretoria, which according to its website is the oldest supplier of agricultural lime in Africa.', ' The company is also the backbone for the personal wealth of the Pistorius family (one member being Oscar Pistorius).']], ['Gordon Day', ['Gordon Raymond Day (born 4 January 1936) is a retired South African sprinter.', ' He competed at the 1960 Summer Olympics in the 400 metres and 4×400 metres relay events and finished fourth in the relay.', ' He was part of the South African team that won the 4×440 yards relay at the 1958 British Empire and Commonwealth Games, while finishing third in the individual 220 yards.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.761\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac2070755429964131be2bd', 'answer': '331 episodes', 'question': 'How many episodes were there of the TV series where Julianna Margulies had the role of Carol Hathaway ?', 'supporting_facts': [['Julianna Margulies', 1], ['ER (TV series)', 0]], 'context': [['The Good Wife', ['The Good Wife is an American legal and political drama television series that aired on CBS from September 22, 2009, to May 8, 2016.', \" The series focuses on Alicia Florrick, the wife of the Cook County State's Attorney, who returns to her career in law after the events of a public sex and political corruption scandal involving her husband.\", ' The series, created by Robert and Michelle King, stars Julianna Margulies, Josh Charles, Christine Baranski, Matt Czuchry, Archie Panjabi, and Alan Cumming, and features Chris Noth in a recurring role.', ' The executive producers are Ridley Scott, Charles McDougall, and David W. Zucker.', ' \"The Good Wife\" is a heavily serialized show featuring several story arcs that carry over several episodes, as well as stand-alone procedural storylines that are concluded by the end of each episode.', ' The serial plots have been especially showcased in its highly praised fifth season.', ' This is a rarity among \"The Good Wife\"\\' s broadcaster CBS, as most of its shows are procedural.']], ['ER (TV series)', ['ER is an American medical drama television series created by novelist and medical doctor Michael Crichton that aired on NBC from September 19, 1994, to April 2, 2009, with a total of 331 episodes spanning over 15 seasons.', ' It was produced by Constant c Productions and Amblin Television, in association with Warner Bros.', ' Television.', ' \"ER\" follows the inner life of the emergency room (ER) of fictional County General Hospital in Chicago, Illinois, and various critical issues faced by the room\\'s physicians and staff.', ' The show became the longest-running primetime medical drama in American television history.', ' It won 23 Primetime Emmy Awards, including the 1996 Outstanding Drama Series award, and received 124 Emmy nominations, which makes it the most nominated drama program in history.', ' \"ER\" won 116 awards in total, including the Peabody Award, while the cast earned four Screen Actors Guild Awards for Outstanding Ensemble Performance in a Drama Series.']], [\"Canterbury's Law\", [\"Canterbury's Law is an American legal drama television series, which aired from March 10 to April 18, 2008 as a mid-season replacement on Fox.\", ' The show was created by Dave Erickson and executive produced by Denis Leary, Jim Serpico, Walon Green, John Kane, and Mike Figgis, who also directed the pilot.', ' The series revolved around Elizabeth Canterbury (portrayed by Julianna Margulies), a rebellious defense attorney willing to bend the law if it protects the wrongfully accused.', ' A rising star, she puts her career on the line to take on risky and unpopular cases, even when they take a toll on her personal life.']], ['Evelyn (film)', ['Evelyn is a 2002 drama film, loosely based on the true story of Desmond Doyle and his fight in the Irish courts (December 1955) to be reunited with his children.', \" The film stars Sophie Vavasseur in the title role, Pierce Brosnan as her father and Aidan Quinn, Julianna Margulies, Stephen Rea and Alan Bates as supporters to Doyle's case.\", ' The film had a limited release in the United States, starting on December 13, 2002 and was later followed by the United Kingdom release on March 21, 2003.']], ['Julianna Margulies', [\"Julianna Margulies ( ; ('Mar-ga-lease') born June 8, 1966) is an American actress and producer.\", ' After several small television roles, Margulies achieved wide recognition for her role as Carol Hathaway on NBC\\'s long-running medical drama \"ER\", for which she won an Emmy Award.', ' She also voiced Neera in \"Dinosaur\" (2000) and appeared in the miniseries \"The Mists of Avalon\" (2001).', ' In 2009, she took on the lead role of Alicia Florrick in the American legal drama \"The Good Wife\" on CBS.', ' Her performance on that show has garnered acclaim: she won the Primetime Emmy Award for Outstanding Lead Actress in a Drama Series twice, a Golden Globe, and a Television Critics Association Award.']], ['The Grid (miniseries)', ['The Grid is a 2004 television miniseries co-produced by the BBC, Fox TV Studios and Carnival Films.', ' It starred Dylan McDermott and Julianna Margulies.', ' It aired on TNT in the US and on BBC Two in the UK over three consecutive nights and is available on DVD in the UK, United States and Australia.', ' It also aired on Seven HD in Australia in 2007.']], ['Carol Hathaway', ['Nurse Carol Hathaway Ross is a fictional character on the popular television show \"ER\", portrayed by Julianna Margulies from 1994 to 2000.', \" Julianna Margulies' removal from the main cast opening credits was in the final episode of season 6.\"]], ['Bang (The Good Wife)', ['\"Bang\" is the fifteenth episode of the first season of the American legal drama television series \"The Good Wife\".', ' It aired on CBS in the United States on March 2, 2010.', ' In the episode, ex States Attorney Peter Florrick (Chris Noth) is released from prison to his home.', ' He is confined by house arrest and starts working on restarting his legal career.', \" His wife Alicia Florrick (Julianna Margulies) has conflicted emotions about his return and distracts herself with a legal case, in which she defends a man of killing a mutual fund manager who assisted in Bernard Madoff's investment scandal.\"]], ['Alicia Florrick', ['Alicia Florrick (née Cavanaugh) is the lead character of CBS television series \"The Good Wife\" and is portrayed by Julianna Margulies, who has received positive reviews for her performance, winning two Primetime Emmy Awards for Outstanding Lead Actress in a Drama Series.']], ['List of The Good Wife episodes', ['The Good Wife is a legal drama television series created by Robert King and Michelle King, which premiered on CBS on September 22, 2009.', ' The show tells the story of Alicia Florrick (Julianna Margulies), whose husband Peter (Chris Noth) has been jailed following a very public sex and corruption scandal.', ' She returns to her old job as a defense attorney under Will Gardner and Diane Lockhart (Josh Charles and Christine Baranski) to rebuild her reputation and provide for her two children, Grace and Zach (Makenzie Vega and Graham Phillips).']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.762\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac2a41e55429967731025d0', 'answer': '1694', 'question': 'What year was the bank famously opposed by Brendan Bracken for co-operation with Adolf Hitler originally established?', 'supporting_facts': [['Brendan Bracken', 0], ['Brendan Bracken', 1], ['Bank of England', 0], ['Bank of England', 1]], 'context': [['Psychopathography of Adolf Hitler', ['The Psychopathography of Adolf Hitler is an umbrella term for psychiatric (pathographic, psychobiographic) literature that deals with the hypothesis that the German Führer and Reichskanzler Adolf Hitler (1889–1945) suffered from mental illness.', ' Both during his lifetime and after his death, Hitler has often been associated with mental disorders such as hysteria, megalomania or paranoid schizophrenia.', ' Psychiatrists and psychoanalysts who have diagnosed Hitler as having mental disturbance include well-known figures such as Walter C. Langer and Erich Fromm.', ' Other researchers, such as Fritz Redlich, have concluded that Hitler probably did not have these disorders.']], ['The Banker', ['The Banker is an English-language monthly international financial affairs publication owned by The Financial Times Ltd. and edited in London.', ' The magazine was first published in January 1926 through founding Editor, Brendan Bracken of the \"Financial News\", who went on to become the chairman of the \"Financial Times\" from 1945-1958.']], ['Johann Ludwig Hannemann', ['Johann Ludwig Hannemann (25 October 1640 – 25 October 1724) was a professor of medicine who famously opposed the idea of the circulation of the blood.', ' He studied the chemistry of phosphorus, gold, and hematite; wrote articles on metallurgy, botany, theology, and various medical topics.', ' He was an adherent of the views of the ancients and pre-Renaissance alchemists.', ' He trained his medical students according to the schools of Galen, Hippocrates, and Aristotle.']], ['Personal standard of Adolf Hitler', ['The personal standard of Adolf Hitler was designed after Reichspräsident Paul von Hindenburg died on 2 August 1934.', ' Adolf Hitler abolished the title \"Reichspräsident\" and in its place instituted the title of \"Führer\" which henceforth could only be used when referring to him personally.', ' Hindenburg used a personal standard consisting of a black eagle on a square gold background edged by a border of black, white and red bands.', ' Hitler decided on 19 August 1934 to adopt a personal standard for himself, which was called \"Personal standard for Adolf Hitler as Leader and Chancellor of the German Nation\".', ' As he was also Supreme Commander of the Armed Forces it was somewhat later known as \"The personal standard for Adolf Hitler as Leader and Supreme Commander of the Armed Forces\" (German: \"Standarte des Führers und Obersten Befehlshabers der Wehrmacht\").']], ['Bank of England', ['The Bank of England, formally the Governor and Company of the Bank of England, is the central bank of the United Kingdom and the model on which most modern central banks have been based.', ' Established in 1694, it is the second oldest central bank in operation today, after the Sveriges Riksbank.', \" The Bank of England is the world's 8th oldest bank.\", \" It was established to act as the English Government's banker and is still one of the bankers for the Government of the United Kingdom.\", ' The Bank was privately owned by stockholders from its foundation in 1694 until it was nationalised in 1946.']], ['Brendan Bracken', ['Brendan Bracken, 1st Viscount Bracken, PC (15 February 1901 – 8 August 1958), was an Irish born businessman and a minister in the British Conservative cabinet.', \" He is best remembered for opposing the Bank of England's co-operation with Adolf Hitler, and for subsequently supporting Winston Churchill's prosecution of World War II against Hitler.\", ' He was also the founder of the modern version of the \"Financial Times\".', ' He served as Minister of Information from 1941 to 1945.']], ['Leibstandarte SS Adolf Hitler order of battle', ['The Leibstandarte SS \"Adolf Hitler\" (LSSAH) was founded in September 1933 as Adolf Hitler\\'s personal Bodyguard formation.', ' It was given the title \"Leibstandarte Adolf Hitler\" (LAH) in November, 1933.', ' On 13 April 1934, by order of Himmler, the regiment became known as the \"Leibstandarte SS Adolf Hitler\" (LSSAH).', ' In 1939 the LSSAH became a separate unit of the Waffen-SS aside the SS-TV and the SS-VT.']], ['The Mind of Adolf Hitler', ['The Mind of Adolf Hitler: The Secret Wartime Report, published in 1972 by Basic Books, is based on a World War II report by psychoanalyst Walter C. Langer which probed the psychology of Adolf Hitler from the available information.', ' The original report was prepared for the Office of Strategic Services (OSS) and submitted in late 1943 or early 1944; it is officially entitled \"A Psychological Analysis of Adolph Hitler: His Life and Legend\".', ' The report is one of two psychoanalytic reports prepared for the OSS during the war in an attempt to assess Hitler\\'s personality; the other is \"Analysis of the Personality of Adolph Hitler\" by the psychologist Henry A. Murray who also contributed to Langer\\'s report.', ' The report eventually became 1000 pages long.']], ['Financial News (1884–1945)', ['The Financial News was a daily British newspaper published in London.', ' It was founded in 1884 by Harry Marks, who had begun on United States newspapers, and set up to expose fraudulent investments.', ' Marks himself was key to the paper\\'s early growth, when it had a buccaneering life fighting against corruption and competing with the \"Financial Times\", but after Marks\\' death it declined.', ' Bought by publishers Eyre & Spottiswoode in 1928 and run by Brendan Bracken, it eventually merged with its great rival in 1945.']], [\"Conspiracy theories about Adolf Hitler's death\", ['Conspiracy theories about Adolf Hitler\\'s death contradict the fact that Adolf Hitler committed suicide in his \"Führerbunker\" on 30 April 1945.', ' Most of these theories hold that Hitler and his wife, Eva Braun, survived and escaped the city of Berlin.', ' While subject to some exposure in popular culture, examples being books such as \"Grey Wolf: The Escape of Adolf Hitler\", these viewpoints are regarded by mainstream historians as disproven fringe theories.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.763\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae625935542995703ce8b2a', 'answer': 'Frank Fertitta, Jr.', 'question': 'Who founded gaming company which owns and operates the Red Rock Resort?', 'supporting_facts': [['Red Rock Casino, Resort &amp; Spa', 0], ['Red Rock Casino, Resort &amp; Spa', 1], ['Station Casinos', 0]], 'context': [['UFC Fight Night 6', ['UFC Fight Night: Sanchez vs. Parisyan (also known as UFC Fight Night 6) was a mixed martial arts event held by the Ultimate Fighting Championship on August 17, 2006.', ' The event took place at the Red Rock Resort Spa and Casino in Las Vegas, Nevada, and was broadcast live on Spike TV in the United States and Canada.', ' It acted as a lead-in to the season four premiere of \"The Ultimate Fighter\".', ' The two-hour broadcast of UFC Fight Night 6 on Spike TV drew a 1.5 overall rating.', '']], ['2012 Libertarian National Convention', [\"The 2012 United States Libertarian National Convention, in which delegates of the Libertarian Party (LP) chose the party's nominees for President of the United States and Vice President of the United States in the 2012 general election, was held May 2–6, 2012 in Las Vegas, Nevada at the Red Rock Resort Spa and Casino.\", ' Former Governor of New Mexico Gary Johnson won the presidential nomination on the first ballot.', ' Retired California state court judge Jim Gray won the vice-presidential nomination, also on the first ballot.', ' The convention also chose to replace most of the Libertarian National Committee party officers and members-at-large.']], ['Red Rock Casino, Resort &amp; Spa', ['Red Rock Resort is a hotel and casino.', ' It is owned and operated by Station Casinos on 70 acre located in Downtown Summerlin in the village of Summerlin Centre in Summerlin, Nevada.', ' Located on Charleston Boulevard, at the interchange of Clark County 215 and some distance from the Las Vegas Strip, the resort is known as a locals casino.', \" It is the flagship property of Station Casinos and the company's corporate headquarters is located on the property.\"]], [\"Hell's Kitchen (U.S. season 2)\", ['Season 2 of \"Hell\\'s Kitchen\" was cast during November 2005, started on June 12, 2006 and concluded on August 14, 2006.', ' The winner was Heather West.', ' Her prize was the position of executive chef at the Red Rock Resort Spa and Casino in Las Vegas with a salary of $250,000, where she worked as chef at the Terra Rossa Restaurant.']], ['Gametrak', ['Gametrak is a brand of 3-dimensional game control systems based on position tracking, designed for home video game platforms such as video game consoles and personal computers.', ' The first Gametrak was invented in 2000 by Elliott Myers, who developed and guided the Gamester video game peripheral range for Leda Media Products and later Radica Games.', ' Myers founded gaming company In2Games around Gametrak in November 2000.']], ['Red Rock, Apache County, Arizona', ['Red Rock (also known as Red Valley) is an unincorporated community and census-designated place (CDP) in Apache County, Arizona, United States.', ' Red Rock is located on the Navajo Nation near the New Mexico border, 16 mi northeast of Lukachukai.', ' Red Rock has a post office with ZIP code 86544; the post office uses the Red Valley name.', ' As of the 2010 census, the Red Rock CDP had a population of 169.']], ['Delta Corp Limited', ['Delta Corp Limited, previously known as Arrow Webtex Ltd., is an Indian real estate, gaming and hospitality corporation that owns and operates casinos and hotels under several brands.', ' The company was founded as Creole Holdings Company Pvt Ltd on 5 November 1990.', ' It is the largest and only gaming and hospitality industry in India.', ' Delta Corp is a public company listed on the Bombay Stock Exchange (BSE) and National Stock Exchange of India Limited (NSE).', ' It runs the majority of the offshore casinos on Panaji, Goa.', ' It is the largest gaming company in India with a revenue of ₹INR 2.39 billion (2016).', ' The company operates casinos in three Indian states: Goa, Daman and Sikkim.']], ['Red Rock River (Montana)', ['The Red Rock River is a roughly 70 mi river in southwestern Montana in the United States.', ' Its drainage basin covers over 1548 mi2 .', \" Its furthest tributary, Hell Roaring Creek, originates in the Beaverhead National Forest within a few hundred meters of the North American Continental Divide and Montana-Idaho border near Brower's Spring, at an elevation of about 9100 ft .\", \" Brower's Spring is near the furthest headwaters of the Missouri River, one of the major watercourses of the central United States.\", ' The drainage flows north and west with its name changing to \"Red Rock Creek\" into the Red Rock Lakes in the middle of a wide grassy valley; the Red Rock River issues from the west side of Lower Red Rock Lake.', ' It flows west, receiving many tributaries such as Peet Creek and Long Creek, widening into the Lima Reservoir and then passing through a canyon, which ends near Lima, Montana.', ' From there, it flows northwest through a valley, passing Kidd and Red Rock, and into Clark Canyon Reservoir.', ' Under the waters of the lake was once the confluence of the Red Rock and Horse Prairie Creek, forming the Beaverhead River, a tributary of the Jefferson River, in turn a headwater of the Missouri River.']], ['Full Tilt Poker Championship at Red Rock', ['Full Tilt Poker Championship at Red Rock (also FullTiltPoker.Net Championship at Red Rock) was a seven-week televised shootout poker tournament played at the Red Rock Resort Spa and Casino in Summerlin near Las Vegas, Nevada.', ' The tournament was sponsored by online poker website Full Tilt Poker and aired by Fox Sports Net.', ' In each of the first six episodes, six professional poker player affiliated with Full Tilt Poker played a single-table freezeout tournament.', ' The winner of each freezeout won US$25,000 and advanced to the seven-handed final table.', ' The seventh seat at the final table was filled by Stefan Rehn, an Internet qualifier.', ' Tournaments featured a speed poker format, with players having 30 seconds to act on their hands with one 60-second time extension per match.']], ['Station Casinos', ['Station Casinos is a gaming company based in the Las Vegas suburb of Summerlin, Nevada, founded by Frank Fertitta, Jr. Station Casinos, along with Affinity Gaming, Boyd Gaming, and American Casino & Entertainment Properties, dominate the locals casino market in Las Vegas.', ' The company purchased several sites that were gaming-entitled, meaning that major casinos can be built at that location without additional approvals.', ' There are only a limited number of such sites available in the Las Vegas area.', ' Station has also branched out into managing casinos that they do not own.', ' Red Rock Resorts, Inc. () is a publicly traded holding company that owns a portion of Station Casinos.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.764\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab913205542991b5579f0e0', 'answer': 'Kentucky Fried Chicken (KFC)', 'question': 'The Sanders-Brown Center on Aging was funded by a grant from a businessman known for what restaurant chain?', 'supporting_facts': [['Sanders–Brown Center on Aging', 2], ['John Y. Brown Jr.', 1]], 'context': [['Prem Ganapathy', ['Prem Ganapathy is an Indian entrepreneur and businessman.', ' He is the founder of the restaurant chain Dosa plaza.', ' Starting with meager investment, he expanded Dosa plaza into a restaurant chain with 45 outlets in India, New Zealand, Oman, and UAE.']], ['KFC', ['KFC, until 1991 known as Kentucky Fried Chicken, is an American fast food restaurant chain that specializes in fried chicken.', \" Headquartered in Louisville, Kentucky, it is the world's second-largest restaurant chain (as measured by sales) after McDonald's, with almost 20,000 locations globally in 123 countries and territories as of December 2015 .\", ' The chain is a subsidiary of Yum!', ' Brands, a restaurant company that also owns the Pizza Hut and Taco Bell chains.']], [\"Nirula's\", [\"Nirula's is India's oldest fast food restaurant chain.\", \" Based in North India and most popular in NCR Delhi, it was Delhi's first fast food restaurant, opening in Connaught Place in 1977.\", ' Today it has over 70 outlets in NCR Delhi, Bihar, Haryana, Madhya Pradesh, Punjab, Rajasthan and Uttar Pradesh states, offering a “Desi” version of Western fast food items.', \" Nirula's success has led them to branch out into other ventures which include, ‘Potpourri’, an Indian cuisine, casual dining restaurant chain; ‘Nirula's 21’, ice cream parlour chain, in addition to pastry shops and two hotels in Noida and Panipat.\", \" Recently Nirula's opened its first franchise in Patna, their first outlet in the entire east zone.\"]], ['David Snowdon', ['David A. Snowdon (born 1952), is an epidemiologist and professor of neurology at the Sanders-Brown Center on Aging at the University of Kentucky.', \" His research interests include antioxidants and aging, and the neuropathology of Alzheimer's disease, especially predictive factors in early life and the role of brain infarction.\"]], ['Chicken in the Rough', [\"Chicken in the Rough, also known as Beverly's Chicken in the Rough, is a fried chicken restaurant chain and former franchise.\", ' It was one of the earliest restaurant chain franchises in the United States.', \" Chicken in the Rough was founded by Beverly and Rubye Osborne in 1936 in Oklahoma City, and the restaurant's specialty half-fried chicken dish was also created in 1936.\", ' The dish itself was also referred to as \"Chicken in the Rough\", and consisted of a half fried chicken, shoestring potatoes and a biscuit with honey.', ' Three restaurants presently serve the dish today, located in Port Huron, Michigan and Canadian neighbor Sarnia, Ontario.', \" The chain's logo was an image of a rooster smoking a cigar and carrying a golf club.\", ' The chain also used a logo of \"Chicken\\'s Caddie\", which depicted a chick acting as a golf caddie, stating \"I\\'ll gladly be fried for Chicken in the Rough\".']], ['Sanders–Brown Center on Aging', ['The Sanders–Brown Center on Aging at the University of Kentucky in Lexington, Kentucky, is responsible for research, education and service programs in aging.', ' The program was founded in 1963 with the creation of the Council on Aging.', ' In 1972, a grant from the Eleanor and John Y. Brown Jr., Foundation led to the construction of the current facility.', ' The center was named for the Browns and his then-business partner, (Col.) Harlan Sanders.', ' The four-story brick and concrete structure fronts South Limestone at the edge of the medical campus.', ' The complex features 32 research laboratories that employs more than 150.']], ['Rochelle Buffenstein', ['Rochelle (Shelley) Buffenstein is a staff scientist at Calico, an Alphabet, Inc. funded research outfit investigating aging.', ' Previously, she had been a professor of Physiology at theBarshop Institute for Longevity and Aging Studies at the University of Texas Health Science Center at San Antonio.', ' Her research focuses on comparative vertebrate physiology, energetics, cancer biology, and aging.', ' She has worked with marsupials, mole-rats, tenrecs, bats, subterranean mammals, and primates.', ' Her best known work involves exceptional aging -- specifically, why naked mole-rats live for so much longer than other rodents.', ' Her lab has investigated theories of aging including oxidative damage theory, the advanced glycation end product theory, and the telomere theory.']], ['Chefette', ['Chefette Restaurants is the largest fast food restaurant chain based in the Caribbean island nation of Barbados.', \" Currently operating throughout the island in 14 locations, Chefette is known for its broasted chicken meals as well as a local curried-'meat + vegetable' (similar to the European Gyro) roll-up or wrap, locally known as a roti.\", ' Chefette was founded by a Trinidadian businessman named Assad John Haloute, who migrated to Barbados in 1971.', ' In 1972, he opened the first Chefette Restaurant at Fontabelle, St. Michael.', ' As the success of the chain grew over the next three decades, the restaurant chain continued its expansion.', \" The company's trademark colours are yellow and purple.\"]], [\"VIP's\", [\"VIP's, alternatively written Vip's, is a defunct restaurant chain in the Western United States that operated from 1968 until the late 1980s, based in Salem, Oregon.\", ' With more than 50 locations, it was once the largest restaurant chain based in Oregon.', ' It was a Denny\\'s-style restaurant, a type that was commonly known at that time as a \"coffee shop\" but is now more commonly known as a casual dining restaurant.', ' Most restaurants were located near freeways and were open 24 hours.', ' At its peak, the chain had locations in five states: Oregon, Washington, Idaho, Nevada and northern California.']], ['John Y. Brown Jr.', ['John Young Brown Jr. (born December 28, 1933) is an American politician, entrepreneur, and businessman from the U.S. state of Kentucky.', ' He served as the 55th governor of Kentucky from 1979 to 1983, although he may be best known for building Kentucky Fried Chicken (KFC) into a multimillion-dollar restaurant chain.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.765\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8f24cc55429924144829e7', 'answer': 'Battle of Dresden', 'question': 'Andrei Ivanovich Gorchakov commanded the 1st Infanty Corps in what major engagement of the Napoleonic Wars?', 'supporting_facts': [['Andrei Ivanovich Gorchakov', 4], ['Battle of Dresden', 0]], 'context': [['VI Corps (Grande Armée)', ['The VI Corps of the \"Grande Armée\" was the name of a French military unit that existed during the Napoleonic Wars.', ' It was formed at the Camp de Boulogne and assigned to Marshal Michel Ney.', \" From 1805 through 1811, the army corps fought under Ney's command in the War of the Third Coalition, the War of the Fourth Coalition, and the Peninsular War.\", ' Jean Gabriel Marchand was in charge of the corps for a period when Ney went on leave.', ' In early 1811, Ney was dismissed by Marshal André Masséna for disobedience and the corps was briefly led by Louis Henri Loison until the corps was dissolved in May 1811.', ' The VI Corps was revived in 1812 for the French invasion of Russia and placed under Laurent Gouvion Saint-Cyr.', ' It entirely consisted of Bavarian soldiers at that time.', ' After the disastrous winter retreat the corps was virtually destroyed.', ' In 1813 during the War of the Sixth Coalition it was recreated with reorganized French troops.', \" Marshal Auguste Marmont took command of the corps and managed it until Emperor Napoleon's abdication in 1814.\", ' It took part in many battles including Dresden and Leipzig in 1813.', ' During the Hundred Days, Georges Mouton, Count de Lobau commanded the VI Corps at the Battle of Waterloo.']], ['Battle of Dresden', ['The Battle of Dresden (26–27 August 1813) was a major engagement of the Napoleonic Wars.', ' The battle took place around the city of Dresden in modern-day Germany.', ' With the recent addition of Austria, the Sixth Coalition felt emboldened in their quest to kick the French out of Central Europe.', ' Despite being heavily outnumbered, French forces under Napoleon scored a modest victory against the Allied army led by Field Marshal Schwarzenberg.', \" However, Napoleon's victory did not lead to the collapse of the coalition, and the lack of effective French cavalry units precluded a major pursuit.\", ' A few days after the battle, the Allies surrounded and captured a French corps at the Battle of Kulm.']], ['III Cavalry Corps (Grande Armée)', ['The III Cavalry Corps (Grande Armée) was a French military formation that fought during the Napoleonic Wars.', ' The corps was created in 1812 and reconstituted in 1813 and 1815.', ' Emperor Napoleon first mobilized the corps for the French invasion of Russia.', ' Commanded by General of Division Emmanuel Grouchy, two divisions of the corps fought at Borodino, Tarutino, and Vyazma.', ' A third division fought at First and Second Polotsk and the Berezina.', ' During the War of the Sixth Coalition in 1813, General of Division Jean-Toussaint Arrighi de Casanova led the corps at Grossbeeren, Dennewitz, Leipzig, and Hanau.', ' During the Hundred Days in 1815, Napoleon reorganized the corps and appointed General of Division François Étienne de Kellermann to lead it.', ' One brigade of the corps was engaged at Quatre Bras and both divisions fought at Waterloo.']], ['IV Cavalry Corps (Grande Armée)', ['The IV Cavalry Corps (Grande Armée) was a French military formation that existed during the Napoleonic Wars.', ' The corps was created in 1812 and rebuilt in 1813 and 1815.', ' Emperor Napoleon first organized the corps for the French invasion of Russia.', ' Under General of Division Victor de Fay de La Tour-Maubourg, the corps fought at Borodino.', ' During the War of the Sixth Coalition in 1813, General of Division François Étienne de Kellermann commanded the all-Polish corps at Leipzig.', ' During the Hundred Days in 1815, Napoleon reconstituted the corps and nominated General of Division Édouard Jean Baptiste Milhaud to direct it.', ' Composed entirely of cuirassier regiments, the two divisions fought at Ligny and Waterloo.']], ['Andrei Ivanovich Gorchakov', ['Andrei Ivanovich Gorchakov (1768 – 1855) led a Russian infantry corps in the German Campaign of 1813 and the French Campaign of 1814 during the Napoleonic Wars.', ' He participated in the 1799 Italian and Swiss expedition on the staff of his uncle Alexander Suvorov and was at Cassano, the Trebbia and Novi.', ' In 1812 he fought at Smolensk and Borodino.', ' At Bautzen in May 1813 he led the second line of the Right Wing.', ' He commanded the 1st Infantry Corps, at Dresden and Leipzig in 1813 and at Bar-sur-Aube, Laubressel and Paris in 1814.']], ['Paul von Radivojevich', ['Paul von Radivojevich (1759 – 15 July 1829) became an army corps commander in the army of the Austrian Empire during the late Napoleonic Wars.', ' He joined the army of the Habsburg Monarchy in 1782 and fought in one of the early battles of the French Revolutionary Wars.', ' He led a Grenz Infantry Regiment before being promoted to general officer in 1807.', ' He led a brigade at Eckmühl in 1809, a division in the summer of 1813, and a corps at Caldiero in 1813 and at the Mincio in 1814.', ' During the 1815 Italian campaign, he led a corps in Switzerland, Piedmont, and France.', ' After the wars, he commanded part of the Military Frontier.', ' He was Proprietor (Inhaber) of an infantry regiment from 1815 until his death in 1829.']], ['Sir George Collier, 1st Baronet', ['Sir George Ralph Collier, 1st Baronet KCB (1774 – 24 March 1824) was an officer of the Royal Navy during the French Revolutionary and Napoleonic Wars, and the War of 1812.', ' He had an eventful early life, being shipwrecked early in his career and later captured by the French.', ' Nevertheless, he saw enough service to attract the attention of powerful patrons that secured his rise through the ranks.', ' An officer of considerable ability, he won a noteworthy victory against a stronger French opponent, before embarking on a period of distinguished service off the Spanish and Portuguese coasts, working closely with the British generals fighting the Peninsular War, and markedly contributing to their success.', ' His good service led to a prime posting in command of a squadron despatched to hunt down and neutralise the American super frigates during the War of 1812.', ' He came close to capturing the  , but lost her in circumstances that were unclear and would later return to haunt him.', ' The years of peace that followed the end of the Napoleonic Wars saw him rewarded with a baronetcy, and his continued to serve in the navy where he was tasked with the suppression of the slave trade.', ' The publishing of William James\\'s account of the War of 1812, which lambasted him for incompetence and cowardice in his failure to catch the \"Constitution\", broke his personal peace.', ' Having failed to clear his name, and increasingly depressed by the accusations, Collier took his own life.']], ['Lucius Curtis', ['Admiral of the Fleet Sir Lucius Curtis, 2nd Baronet, KCB, DL (3 June 1786 – 14 January 1869) was a senior officer of the Royal Navy during the nineteenth century.', \" The son of Sir Roger Curtis, 1st Baronet, Lord Howe's flag captain at the Glorious First of June, Lucius served during the Napoleonic Wars and was heavily involved in the Mauritius campaign of 1810.\", ' During this campaign, Curtis commanded the frigate HMS \"Magicienne\" with the blockade squadron under Josias Rowley and was still in command when the ship was destroyed at the Battle of Grand Port.', ' \"Magicienne\" grounded on a coral reef early in the engagement and despite the best efforts of Curtis and his crew, the ship had to be abandoned, Curtis setting her on fire to prevent her subsequent capture.']], ['HMS Cleopatra (1779)', ['HMS \"Cleopatra\" was a 32-gun \"Amazon\"-class fifth rate frigate of the Royal Navy.', ' She had a long career, seeing service during the Fourth Anglo-Dutch War, and the French Revolutionary and Napoleonic Wars.', ' During the latter wars she fought two notable engagements with larger French opponents.', ' In the first engagement she was forced to surrender, but succeeded in damaging the French ship so badly that she was captured several days later, while the \"Cleopatra\" was retaken.', ' In the second she forced the surrender of a 40-gun frigate.', ' After serving under several notable commanders she was broken up towards the end of the Napoleonic Wars.']], ['HMS Cherub (1806)', ['HMS \"Cherub\" was an 18-gun Royal Navy \"Cormorant\"-class sloop built in Dover in 1806.', ' She participated in two major campaigns in the West Indies during the Napoleonic Wars, and one major engagement in the Pacific during the War of 1812, all each of which earned her crews clasps to the Naval General Service Medal.', ' The Navy sold her in 1820.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.766\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac07327554299012d1db5f2', 'answer': 'an American football quarterback', 'question': 'Who is Colin Kaepernick and what is his preferred nickname?', 'supporting_facts': [['Kaep (disambiguation)', 0], ['Colin Kaepernick', 0]], 'context': [['Wickedness Preferred', ['Wickedness Preferred is a lost 1928 American comedy silent film directed by Hobart Henley and written by Colin Clements, Robert E. Hopkins and Florence Ryerson.', ' The film stars Lew Cody, Aileen Pringle, Mary McAllister, Bert Roach and George K. Arthur.', ' The film was released on January 28, 1928, by Metro-Goldwyn-Mayer.']], ['Lotus Mark VI', [\"After building multiple trials and road racing cars, Colin Chapman introduced his first 'production' car, the Lotus Mark VI, in 1952.\", ' The heart of the Mark VI was a space frame chassis.', ' Rather than a complete car, it was available to the general public as kit, wherein the customer could install any preferred engine and gearbox, making it eligible for a wider number of formulae.']], ['Swing Around the Circle', ['Swing Around the Circle refers to a disastrous speaking campaign undertaken by U.S. President Andrew Johnson between August 27 and September 15, 1866, in which he tried to gain support for his mild Reconstruction policies and for his preferred candidates (mostly Democrats) in the forthcoming midterm Congressional elections.', ' The tour received its nickname due to the route that the campaign took: \"Washington, D.C., to New York, west to Chicago, south to St. Louis, and east through the Ohio River valley back to the nation\\'s capital\".']], ['Kaep (disambiguation)', ['Kaep is a nickname for athlete Colin Kaepernick (although he prefers \"Kap\").', ' It may also refer to:']], ['Toni Kallio', ['Toni Kallio (born 9 August 1978) is a Finnish former footballer who last presented Ilves in Ykkönen.', ' His preferred position is left back, but he can also operate as centre back and used to play as forward when he joined HJK.', ' His nickname is \"Bonecrusher\", coming from his great physical presence and playing style.']], ['Proposition Joe', ['Joseph Stewart, better known as \"Proposition Joe\" or \"Prop Joe\", is a fictional character on the HBO drama \"The Wire\", played by actor Robert F. Chew.', ' Joe was an Eastside drug lord who preferred a peaceful solution to business disputes when possible.', ' He was responsible for creating the lucrative New Day Co-Op with Stringer Bell, supplying much of Baltimore with heroin brought into the city by \"The Greeks\".', ' Joe was a portly and amiable presence, but was often a match in wits for rival drug lords Avon Barksdale and Marlo Stanfield, and was able to manipulate most situations to his advantage.', ' His nickname stemmed from his trademark phrase \"I\\'ve got a proposition for you\", going back to his days selling test answers on the school yard.', ' Along with Poot Carr, Wee-Bey Brice, Omar Little, and Bubbles, he is one of the few characters from the drug trade to appear in every season.']], ['Gregory Mcdonald', ['Gregory Mcdonald (February 15, 1937 – September 7, 2008) was an American mystery writer best known for his creation of the character Irwin Maurice Fletcher, an investigative reporter who preferred the nickname \"Fletch.\"']], ['Colin Kaepernick', ['Colin Rand Kaepernick ( ; born November 3, 1987) is an American football quarterback who is currently a free agent.', ' Kaepernick played college football at the University of Nevada, where he was named the Western Athletic Conference (WAC) Offensive Player of the Year twice and became the only player in NCAA Division I FBS history to amass 10,000 passing yards and 4,000 rushing yards in a career.', ' After graduating, he was selected by the San Francisco 49ers in the second round of the 2011 NFL Draft.']], ['Cory Lopez', ['Cory Lopez is an elite professional surfer born on March 21, 1977 in Dunedin, Florida, USA.', ' Lopey is his preferred nickname.', \" Cory has been a top ranked contender on the ASP World Surfing circuit (ASP World Tour) for multiple years and is considered by many to be one of the best 'Free Surfers' on the planet.\"]], ['Willie &quot;Two-Knife&quot; Altieri', ['Willie \"Two-Knife\" Altieri, (4 Mar 1891- Oct 1970?)', ' was a New York gangster who served as the chief enforcer for Frankie Yale\\'s Italian-American \"Black-Hand\" gang, one of the most powerful criminal organizations in 1920\\'s New York City.', ' He got his nickname after his preferred method of dispatching a victim.', ' Willie had killed dozens of rival gangsters during the 1920s and was considered an important figure in the \"Black-Hand\" gang.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.766\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a88a5645542997e5c09a679', 'answer': 'Take It Easy', 'question': 'Which was published first, Take It Easy or Personal Preference?', 'supporting_facts': [['Take It Easy (game)', 0], ['Take It Easy (game)', 1], ['Personal Preference', 0]], 'context': [['Järvenpää Plus', ['Järvenpää Plus is a local political party in the municipality of Järvenpää, Finland.', ' It was founded as Järvenpää 2000, but changed its name to Järvenpää 2000+, and in the beginning of 2012, changed its name to \"Järvenpää Plus\".', ' It first participated in the municipal elections in 1988 together with Greens and unaligned candidates.', ' In the 2004 municipal elections the party got 2186 votes (14.2%).', ' It won seven seats in the municipal council.', ' Its most popular candidate was Rauha-Maria Mertjärvi, who got 520 personal preference votes.']], ['Jianbing', ['Jianbing () is a traditional Chinese street food similar to crepes.', ' It is a type of \"bing\" generally eaten for breakfast and hailed as \"one of the China\\'s most popular street breakfasts.\"', ' The main ingredients of jianbing are a batter of wheat and grain flour, eggs and sauces.', ' It can be topped with different fillings and sauces such as \"buocui\" (薄脆 crispy fried cracker), chopped or diced mustard pickles, scallions and coriander, chili sauce or hoisin sauce depending on personal preference.', ' It is often folded several times before serving.']], ['Personal Preference', ['Personal Preference is a 1987 board game created by Donal Carlston that involves guessing the order in which a player prefers foods, activities, people, and other items compared to one another.', ' The game was published by Brøderbund Games in the United States, Playtoy Industries in Canada, and Parker Brothers International in Britain.']], ['Personal Taste', ['Personal Taste (; lit.', ' \"Kae-in\\'s Taste\" or \"Kae-in\\'s Preference\"; also known as Personal Preference) is a 2010 South Korean television series, starring Son Ye-jin and Lee Min-ho.', \" It is adapted from Lee Se-in's 2007 novel of the same name about a furniture designer, Park Kae-in, who lives together with architect Jeon Jin-ho under the mistaken assumption that he's gay.\", ' It aired on MBC from March 31 to May 20, 2010 on Wednesdays and Thursdays at 21:55 for 16 episodes.']], ['Pie floater', ['The pie floater (also known as a floater and a pea floater), is an Australian dish particularly common in Adelaide and, to a lesser degree, Sydney.', ' It commonly consists of a traditional Australian-style meat pie, usually sitting, but sometimes submerged (sometimes upside down) in a bowl of thick pea soup made from Blue boiler peas .', ' It is usually garnished with tomato sauce, and the consumer may also add mint sauce, salt, pepper and/or malt vinegar to personal preference.', ' The pie floater is usually purchased in the street from pie-carts as a late evening meal.']], ['Take It Easy (game)', ['Take It Easy is an abstract strategy board game created by Peter Burley.', ' It can be characterized as a spatial bingo-like game, and has been published by Ravensburger and subsequently by several other publishers since 1983.', ' Each player gets a board with places for 19 hexagon tiles to place in a hexagon shape.', ' Additionally, players get identical sets of tiles which have different types of colored/numbered lines crossing in three directions.', ' One player draws a tile randomly and then tells the others which he drew.', ' Each player then puts their matching tile on their board in any available spot.', ' This is repeated until the board is filled.', ' The object is to complete same colored/numbered lines across your board, for which points are scored according to the numbers on those lines.', ' The maximum score possible is 307.']], ['Sexual racism', ['Sexual racism is the \"sexual rejection of the racial minority, the conscious attempt on the part of the majority to prevent interracial cohabitation.\"', ' It is the discrimination between potential sexual or romantic partners on the basis of perceived racial identity.', ' However, not everyone agrees that this should be classified as racism, some argue that distinguishing among partners on the basis of perceived race is not racism at all but a justifiable personal preference.', ' The origins of sexual racism can be explained by looking at its history, especially in the USA, where the abolition of slavery and the Reconstruction Era had significant impacts on interracial mixing.', ' Attitudes towards interracial relationships, and indeed marriage, have increased in positivity in the last 50 years.', ' In 1968, 73% of US citizens disapproved of the right to marry inter-racially, whereas this figure dropped to 17% by 2007, this illustrating the reduction in discriminatory attitudes towards interracial dating.', ' Irrespective of this, there still remains the issue of sexual racism in the online dating world, in that preferences appear to follow a racial hierarchy.', \" The exclusion of races dissimilar to one's own is a main feature of sexual racism, however a reluctance to date inter-racially predominantly spans from the discriminatory views often possessed by those in society, as opposed to purely a same-race individual preference.\", ' Moreover, this racial discrimination also deviates into the form of the sexual dehumanisation of individuals of other racial identities.', \" Sharing the basic premise, originating from the 'taboo' nature of interracial relations, individuals of other racial groups are classified as forbidden sexual objects; the result of a racial fetish.\", ' This sexualised reductionism is, concurrently, a form of sexual racism.']], ['Spiritual wifery', ['Spiritual wifery is a term first used in America by the Immortalists in and near the Blackstone Valley of Rhode Island and Massachusetts in the 1740s.', ' The term describes the idea that certain people are divinely destined to meet and share their love (at differing points along the carnal-spiritual spectrum, depending on the particular religious movement involved) after a receiving a spiritual confirmation, and regardless of previous \"civil\" marital bonds.', ' Its history in Europe among various Christian primitivistic movements has been well documented.', ' The followers of Jacob Cochran as early as 1818 used \"spiritual wifery\" to describe their religious doctrine of free love.', ' Often confused with polygamy, spiritual wifery among the Cochranites was the practice in which communal mates were temporarily assigned and reassigned, either by personal preference or religious authority.']], ['Fishing rod tapers', ['Fishing rod tapers describe how much a fishing rod bends or flexes under pressure.', ' Different tapers are used for different fishing scenarios as well as for personal preference.']], [\"Song Ji-hyo's Beauty View\", [\"Song Ji-hyo's Beauty View (), is a South Korean television program on JTBC2 hosted by Song Ji-hyo, Gong Myung and beauty editor Kim Mi Gu.\", ' The show provides a perfect beauty guide to match the personal preference.', ' It was used to air on every Thursday at 9.20pm KST on JTBC2.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.767\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ade82ba5542992fa25da7a3', 'answer': 'no', 'question': 'Are Catasetum and Origanum in the same family?', 'supporting_facts': [['Catasetum', 0], ['Origanum', 0]], 'context': [['Origanum laevigatum', ['Origanum laevigatum is a species of flowering plant in the family Lamiaceae, native to Cyprus, Syria, and Turkey.', ' It is a woody-based perennial growing to 50 - tall by 45 cm wide, with strongly aromatic leaves, and loose clusters of pink funnel-shaped flowers with persistent purple bracts, throughout the summer.']], ['Origanum libanoticum', ['Origanum libanoticum (Lebanese oregano, hopflower oregano, cascading hopflower oregano, ornamental oregano or cascading oregano) is a species of herbaceous flowering plant in the family Lamiaceae, native to the mountains of Lebanon and Syria.']], ['Oregano', ['Oregano ( or ; ;, scientific name \"Origanum vulgare\", is a flowering plant in the mint family (Lamiaceae).', ' It is native to temperate western and southwestern Eurasia and the Mediterranean region.']], ['Origanum rotundifolium', ['Origanum rotundifolium, the round-leaved oregano, is a species of flowering plant in the family Lamiaceae, native to Turkey, Armenia and Georgia.', ' It is a small woody-based perennial or subshrub growing to 10 - tall by 30 cm wide, with strongly aromatic leaves, and loose clusters of pink flowers with hop-like pale green bracts, throughout the summer.']], ['Origanum amanum', ['Origanum amanum, the Amanum oregano, is a species of flowering plant in the family Lamiaceae, native to the Hatay Province of southern Turkey, bordering on Syria.', ' It is an evergreen subshrub growing to 10 - tall by 30 cm wide, with strongly aromatic leaves, and clusters of pink funnel-shaped flowers in summer and autumn.']], ['Catasetum', ['Catasetum, abbreviated as Ctsm in horticultural trade, is a genus of showy epiphytic Orchids, family Orchidaceae, subfamily Epidendroideae, tribe Cymbidieae, subtribe Catasetinae, with 166 species, many of which are highly prized in horticulture.']], [\"Za'atar\", ['Za\\'atar (Arabic: زَعْتَر\\u200e \\u200e , ] ) is a generic name for a family of related Middle Eastern herbs from the genera \"Origanum\" (oregano), \"Calamintha\" (basil thyme), \"Thymus\" (typically \"Thymus vulgaris\", i.e., thyme), and \"Satureja\" (savory).', ' The name \"za\\'atar\" alone most properly applies to \"Origanum syriacum\", considered in biblical scholarship to be the hyssop (Hebrew: אזוב\\u200e \\u200e ] ) of the Hebrew Bible.', ' It is also the name for a condiment made from the dried herb(s), mixed with sesame seeds, dried sumac, and often salt, as well as other spices. Used in Levantine cuisine, both the herb and spice mixture are popular throughout the Middle East.']], ['Origanum syriacum', ['Origanum syriacum; syn.', ' Majorana syriaca (also Origanum maru, although this primarily refers to a hybrid of \"O. syriacum\"), bible hyssop, Biblical-hyssop, Lebanese oregano or Syrian oregano, is an aromatic perennial herb in the mint family, Lamiaceae.']], ['Origanum', ['Origanum ( )is a genus of herbaceous perennials and subshrubs in the family Lamiaceae, native to Europe, North Africa, and much of temperate Asia, where they are found in open or mountainous habitats.', ' A few species also naturalized in scattered locations in North America and other regions.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.767\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7502715542996c70cfae74', 'answer': 'Thon Marial Maker', 'question': 'Which Sudanese-born Australian professional basketball player for the Milwaukee Bucks exemplifies that players do not have to play at least a year in college basketball before being drafted into the NBA?', 'supporting_facts': [['NBA high school draftees', 0], ['NBA high school draftees', 3], ['Thon Maker', 0]], 'context': [['Matthew Dellavedova', ['Matthew Dellavedova (born 8 September 1990) is an Australian professional basketball player for the Milwaukee Bucks of the National Basketball Association (NBA).', \" He played college basketball for Saint Mary's College of California and has played on the Australia national team.\", ' Dellavedova won an NBA championship with the Cleveland Cavaliers in 2016.']], ['Darington Hobson', [\"Darington O'Neal Hobson (born September 29, 1987) is an American professional basketball player for the Guangxi Weizhuang Rhinos of the Chinese National Basketball League (NBL).\", \" He played college basketball for the University of New Mexico Lobos men's basketball team.\", ' Born in Las Vegas, Nevada, Hobson attended five high schools and a junior college before finally becoming eligible to play Division I college basketball.', ' Hobson was drafted in the 2nd round (37th overall) of the 2010 NBA Draft by the Milwaukee Bucks.', ' Hobson was waived on December 2, 2010, due to injury.', ' A year later, Hobson was re-signed by the Bucks for the 2011–12 season.', ' He was waived again on February 3, 2012.']], ['Kareem Abdul-Jabbar', ['Kareem Abdul-Jabbar (born Ferdinand Lewis Alcindor Jr.; April 16, 1947) is an American retired professional basketball player who played 20 seasons in the National Basketball Association (NBA) for the Milwaukee Bucks and the Los Angeles Lakers.', ' During his career as a center, Abdul-Jabbar was a record six-time NBA Most Valuable Player (MVP), a record 19-time NBA All-Star, a 15-time All-NBA selection, and an 11-time NBA All-Defensive Team member.', ' A member of six NBA championship teams as a player and two as an assistant coach, Abdul-Jabbar twice was voted NBA Finals MVP.', ' In 1996, he was honored as one of the 50 Greatest Players in NBA History.', ' NBA coach Pat Riley and players Isiah Thomas and Julius Erving have called him the greatest basketball player of all time.']], ['D. J. Wilson', ['DeVante Jaylen \"D. J.\" Wilson (born February 19, 1996) is an American basketball player for the Milwaukee Bucks of the National Basketball Association (NBA).', ' He played college basketball for the Michigan Wolverines and completed his junior season for the 2016–17 team.', ' He was drafted 17th overall in the 2017 NBA draft by the Milwaukee Bucks.']], ['Sidney Moncrief', ['Sidney A. Moncrief (born September 21, 1957) is an American retired professional basketball player.', ' As an NCAA college basketball player from 1975 to 1979, Moncrief played for the University of Arkansas Razorbacks from 1975 to 1979, leading them to the 1978 Final Four and a win in the NCAA Consolation Game versus #6 Notre Dame.', ' Nicknamed Sid the Squid, Sir Sid, and El Sid, Moncrief went on to play 11 seasons in the National Basketball Association, including ten seasons with the Milwaukee Bucks.', ' He was a five-time NBA All-Star and won the first two NBA Defensive Player of the Year awards in 1983 and 1984.']], ['Dan Langhi', ['Daniel Matthew Langhi (born November 28, 1977) is an American former professional basketball player.', ' Born in Chicago, Illinois, he was raised in the small western Kentucky town of Benton.', ' In addition to his high school basketball career, where he finish as the runner-up for Kentucky\\'s prestigious \"Mr. Basketball\" award, Langhi won regional titles as a member of Marshall County\\'s soccer teams.', ' After growing six inches during his sophomore year of high school, he joined his two older brothers in playing college basketball, signing to play college basketball at Vanderbilt, and was drafted 31st overall by the Dallas Mavericks in the second round of the 2000 NBA Draft.', ' Langhi played for the Houston Rockets, the Phoenix Suns, the Golden State Warriors and the Milwaukee Bucks in the NBA.']], ['Malcolm Brogdon', ['Malcolm Moses Adams Brogdon (born December 11, 1992) is an American professional basketball player for the Milwaukee Bucks of the National Basketball Association (NBA).', ' He played college basketball for the Virginia Cavaliers under Tony Bennett.', ' As a senior in 2015–16, he was named the ACC Player of the Year and ACC Defensive Player of the Year, becoming the first player in conference history to earn both honors in the same season.', ' He was selected in the second round of the 2016 NBA draft by the Bucks with the 36th overall pick.', ' He went on to win the NBA Rookie of the Year Award, becoming the first second-round pick in the NBA draft lottery era to do so.']], ['NBA high school draftees', ['The NBA high school draftees are players who have been drafted to the National Basketball Association (NBA) straight out of high school without playing basketball at the collegiate level.', ' The process of jumping directly from high school to the professional level is also known as going prep-to-pro.', ' Since 2006, the practice of drafting high school players has been prohibited by the new collective bargaining agreement, which requires that players who entered the draft be 19 years of age and at least one year removed from high school.', ' Contrary to popular belief, the player does not have to play at least a year in college basketball, as the player can choose to instead play in another professional league (especially overseas) like Brandon Jennings or Emmanuel Mudiay in Italy and China respectively, simply take the year off, such as the case with Satnam Singh Bhamara, or even hold themselves back a year in high school before declaring for the draft, such as the case with Thon Maker.']], ['Andrew Bogut', ['Andrew Michael Bogut (born 28 November 1984) is an Australian professional basketball player for the Los Angeles Lakers of the National Basketball Association (NBA).', ' The 7 ft center was selected by the Milwaukee Bucks with the first overall pick in the 2005 NBA draft.', ' He earned All-NBA Third Team honors with the Bucks in 2010.', ' He was traded to the Golden State Warriors in 2012, and was named NBA All-Defensive Second Team in 2015, when he won an NBA championship with the Warriors.']], ['Thon Maker', ['Thon Marial Maker (born 25 February 1997) is a Sudanese-born Australian professional basketball player for the Milwaukee Bucks of the National Basketball Association (NBA).', \" He attended high school at Orangeville District Secondary School and played basketball for Canada's Athlete Institute.\", ' Coming out of high school, Maker was considered a five-star recruit by most basketball recruiting services.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.768\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab70c41554299110f219a9d', 'answer': 'John Snow', 'question': 'What is the name of the physician who studied the cause of the 1854 Broad Street cholera outbreak in London, England?', 'supporting_facts': [['1854 Broad Street cholera outbreak', 0], ['1854 Broad Street cholera outbreak', 1], ['John Snow', 0]], 'context': [['2012 Sierra Leonean cholera outbreak', ['s of 24 September 2012 , a cholera outbreak in Sierra Leone has caused the deaths of 392 people.', \" It is the country's largest outbreak of cholera since first reported in 1970 and the deadliest since the 1994–1995 cholera outbreak.\", ' The outbreak has also affected Guinea, which shares a reservoir near the coast.', ' This is the largest cholera outbreak in Africa in 2012.']], ['1854 Broad Street cholera outbreak', ['The Broad Street cholera outbreak (or Golden Square outbreak) was a severe outbreak of cholera that occurred in 1854 near Broad Street (now Broadwick Street) in the Soho district of London, England.', \" This outbreak, which killed 616 people, is best known for the physician John Snow's study of its causes and his hypothesis that contaminated water, not air, was the source of cholera.\", ' This discovery came to influence public health and the construction of improved sanitation facilities beginning in the mid-19th century.', ' Later, the term \"focus of infection\" would be used to describe sites, such as the Broad Street pump, in which conditions are good for transmission of an infection.', \" John Snow's endeavor to find the cause of the transmission of cholera caused him to unknowingly create a double-blind experiment.\"]], ['Cholera Hospital', ['Cholera Hospital was established on June 24, 1854, at Franklin Street in New York City.', ' The institution was built to treat cholera patients who were denied admittance to City Hospital in Manhattan during an onset of the disease in the summer of 1854.', ' The Mayor of New York, Jacob Westervelt, and the New York City Commissioners, took control of the building at 105 Franklin Street in anticipation of an eminent cholera epidemic.', ' A few weeks afterward a second hospital for cholera sufferers was opened at a schoolhouse on Mott Street (Manhattan).', ' A book published by a New York physician in 1835 shows that a hospital called the Duane-Street Cholera Hospital existed in New York as early as 1835, but the relationship between the Duane-Street hospital and the Cholera Hospital at Franklin Street is unclear.']], ['1853 Copenhagen cholera outbreak', ['The 1853 Copenhagen cholera outbreak was a severe outbreak of cholera which occurred in Copenhagen, Denmark in 1853 as part of the third cholera pandemic.', ' It killed about 4,800 people.']], ['2009 Papua New Guinea cholera outbreak', ['The Papua New Guinea cholera outbreak is an outbreak of cholera along the Northern Coast of Papua New Guinea.', \" It is the country's first outbreak of cholera in 50 years, and is currently spreading across the country, raising fears of an epidemic.\"]], ['1881–96 cholera pandemic', ['The fifth cholera pandemic (1881–96) was the fifth major international outbreak of cholera in the 19th century starting in India.', ' It spread throughout Asia and Africa, and reached parts of France, Germany, Russia, and South America.', ' The 1892 outbreak in Hamburg, Germany was the only major European outbreak; about 8,600 people died in that city.', ' Although many residents held the city government responsible for the virulence of the epidemic, it continued with practices largely unchanged.', ' This was the last serious European cholera outbreak of the century.']], ['2016–17 Yemen cholera outbreak', ['In October 2016, an outbreak of cholera began in Yemen.', ' The outbreak is \"unprecedented scale,\" according to the World Health Organization (WHO) and is ongoing as of 30 September 2017.', ' The Cholera outbreak is a result of the ongoing war led by Saudi led coalition and Houthis in Yemen since March 2015.', ' As stated by in the statement of the UNICEF and WHO esecutive directors: \"This deadly cholera outbreak is the direct consequence of two years of heavy conflict.', ' Collapsing health, water and sanitation systems have cut off 14.5 million people from regular access to clean water and sanitation, increasing the ability of the disease to spread.', ' Rising rates of malnutrition have weakened children’s health and made them more vulnerable to disease.', ' An estimated 30,000 dedicated local health workers who play the largest role in ending this outbreak have not been paid their salaries for nearly 10 months\"']], ['The Ghost Map', [\"The Ghost Map: The Story of London's Most Terrifying Epidemic\\xa0– and How it Changed Science, Cities and the Modern World is a book by Steven Berlin Johnson in which he describes the most intense outbreak of cholera in Victorian London (See 1854 Broad Street cholera outbreak) The book incorporated the idea of gemeinschaft, dealing with the effects of an epidemic in a city of common values, language, and traditions.\", ' The two central protagonists are Dr. John Snow, who created a map of the cholera cases, and the Reverend Henry Whitehead, whose extensive knowledge of the local community helped determine the initial cause of the outbreak.', ' Dr. John Snow was a revered anesthetist who carried out epidemiological work in Soho, London.', ' Around the mid-1850s Snow figured out the source of cholera contamination to be the drinking water from the Broad Street pump.']], ['2008 Zimbabwean cholera outbreak', ['The 2008 Zimbabwean cholera outbreak was an epidemic of cholera affecting much of Zimbabwe from August 2008 until June 2009.', ' The outbreak began in Chitungwiza in Mashonaland East Province in August 2008, then spread throughout the country so that by December 2008, cases were being reported in all 10 provinces.', ' In December 2008, The Zimbabwean government declared the outbreak a national emergency and requested international aid.', ' The outbreak peaked in January 2009 with 8,500 cases reported per week.', ' Cholera cases from this outbreak were also reported in neighboring countries South Africa, Malawi, Botswana, Mozambique, and Zambia.', ' With the help of international agencies, the outbreak was controlled, and by July 2009, after no cases had been reported for several weeks, the Zimbabwe Ministry of Health and Child Welfare declared the outbreak over.', ' In total, 98,596 cases of cholera and 4,369 deaths were reported, making this the largest outbreak of cholera ever recorded in Zimbabwe.', ' The large scale and severity of the outbreak has been attributed to poor sanitation, limited access to healthcare, and insufficient healthcare infrastructure throughout Zimbabwe.']], ['John Snow', ['John Snow (15 March 1813 – 16 June 1858) was an English physician and a leader in the adoption of anaesthesia and medical hygiene.', ' He is considered one of the fathers of modern epidemiology, in part because of his work in tracing the source of a cholera outbreak in Soho, London, in 1854.', ' His findings inspired fundamental changes in the water and waste systems of London, which led to similar changes in other cities, and a significant improvement in general public health around the world.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.769\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab692b5554299710c8d1ed6', 'answer': 'Hernán Crespo', 'question': '2002–03 Inter Milan season was marked by the arrival of what retired Argentine footballer?', 'supporting_facts': [['2002–03 Inter Milan season', 0], ['Hernán Crespo', 0]], 'context': [['2002–03 Inter Milan season', ['The start of the season was marked by the departure of Ronaldo and the arrival of Hernán Crespo after club had already acquired Fabio Cannavaro, Matias Almeyda and Domenico Morfeo.', ' Crespo, along with Vieri, built an attacking duo.', ' Crespo was essential in the 2002–03 UEFA Champions League while Vieri usually scored in the domestic matches.', ' Their partnership worked until Crespo sustained an injury, which sidelined him for several weeks.', ' Without him, despite replaced by Batistuta, Inter lost some key matches.', ' One of these was against Juventus who, could finally aim for the title.']], ['Trofeo Pirelli', ['The Pirelli Cup is an annual friendly football tournament sponsored by the Pirelli Tyre company.', ' The competition was started in 1996 as a single 90-minute friendly match between Pirelli-sponsored Inter Milan and another invited team.', ' Since 1996, 14 editions of the one-match tournament have been held.', ' The latest edition of the competition was held in 2010 in Baltimore, featuring Inter Milan and Manchester City.', ' The match ended 3-0 as Inter won the trophy.']], [\"Sergio D'Autilia\", [\"Sergio D'Autilia (born 20 August 1977) is an Italian footballer who plays as a forward for Brera Calcio.\", ' He made his fully professional club debut for Inter in the 1996–97 Coppa Italia on November 6, 1996 against Cagliari.', ' While he never played a league match with Inter, he had 19 appearances on the bench during the 1996–97 Inter Milan season and also in the second leg of the 1997 UEFA Cup Final against FC Schalke 04.']], ['Luis Artime', ['Luis Artime (born 2 December 1938 in Parque Civit in Mendoza Province) is a retired Argentine footballer, who played as a striker, and scored more than 1000 goals during his career.', ' His son Luis Fabián Artime is also a retired Argentine footballer who played in the 1990s.']], ['List of Inter Milan managers', ['Inter Milan is an Italian association football club based in Milan, Lombardy.', ' The club was formed on 9 March 1908 to allow the foreign playes to play in Italy.', ' Inter played its first competitive match on 10 January 1910 against their cross-town rivals Milan, in which they lost 3–2.', ' The club won its very first title in 1910 – the 1909–10 Italian Football Championship.', ' Since then, the club has won further 17 league titles, along with seven Coppa Italia and five Supercoppa Italiana.', ' They have also been crowned champions of Europe on three occasions by winning two European Cups back-to-back in 1964 and 1965 and then another in 2010.', ' The club experienced the most successful period in their history from 2006 to 2010, in which it won five successive league titles, equaling the all-time record at that time, by adding three Italian Cups, three Italian Supercups, one UEFA Champions League and one FIFA Club World Cup.', ' During the 2009–10, Inter become the first and only Italian team to win the Treble and the second team to win five trophies in a calendar year.']], ['Hernán Crespo', ['Hernán Jorge Crespo (] ; born 5 July 1975) is a retired Argentine footballer, current coach and the marquee player for the \"Kolkata 5s\" Futsal team in the Premier Futsal league.', ' A prolific striker, he has scored over 300 goals in a career spanning 19 years.', \" At international level, Crespo scored 35 goals and is Argentina's third highest goalscorer behind only Gabriel Batistuta and Lionel Messi.\", ' He played in three FIFA World Cups: 1998, 2002, 2006.']], ['List of Inter Milan honours', ['This is a list of Inter Milan honours.', ' Inter Milan is an Italian football club and this page contains historical and current trophies pertaining to the club.']], ['List of Inter Milan players', ['Inter Milan is an Italian association football club based in Milan, Lombardy.', ' The club was formed on 9 March 1908 to allow the foreign playes to play in Italy.', ' Inter played its first competitive match on 10 January 1910 against their cross-town rivals Milan, in which they lost 3–2.', ' The club won its very first title in 1910 – the 1909–10 Italian Football Championship.', ' Since then, the club has won further 17 league titles, along with seven Coppa Italia and five Supercoppa Italiana.', ' They have also been crowned champions of Europe on three occasions by winning two European Cups back-to-back in 1964 and 1965 and then another in 2010.', ' The club experienced the most successful period in their history from 2006 to 2010, in which it won five successive league titles, equaling the all-time record at that time, by adding three Italian Cups, three Italian Supercups, one UEFA Champions League and one FIFA Club World Cup.', ' During the 2009–10, Inter become the first and only Italian team to win the Treble and the second team to win five trophies in a calendar year.']], ['Luis Suárez (footballer, born 1935)', ['Luis Suárez Miramontes (] ; born 2 May 1935), also known by the diminutive Luisito, is a Spanish former footballer and manager.', ' He played as a midfielder for Deportivo de La Coruña, CD España Industrial, FC Barcelona, Inter Milan, Sampdoria and Spain.', \" Suárez is regarded as one of Spain's greatest players; he was noted for his elegant, fluid, graceful style of play.\", ' Nicknamed \"El Arquitecto\" (The Architect) he was noted for his perceptive passing and explosive shot and in 1960 he became the only Spanish-born player to be voted Ballon d\\'Or.', ' In 1964 he helped Spain win the European Championship.', ' Suarez originally achieved prominence as a creative inside forward or attacking midfielder for the great Barcelona team of the 1950s before he joined Inter Milan where he reached his prime as deep lying playmaker for the legendary \"Grande Inter\" team of the 1960s.', \" He played a pivotal role in the success Herrera's Inter Milan side, and was one of the primary creative forces in the squad, due to his ball skills, vision, and passing range.\", ' He retired as a player in 1973, after three seasons at Sampdoria.']], ['Camillo Achilli', ['Camillo Achilli (21 August 1921 – 14 June 1998) was a professional Italian footballer who played for Inter Milan and Genoa.', ' After retiring as a player in 1953, Achilli enjoyed a career as a coach, managing sides such as Lecco, Inter Milan and Palermo.', ' His son was Marco Achilli.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.770\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adf6aca5542993344016c8d', 'answer': 'Jackson', 'question': 'Ocean Springs School District is located in the city in what Mississippi county?', 'supporting_facts': [['Ocean Springs School District', 0], ['Ocean Springs, Mississippi', 0]], 'context': [['Ocean Springs High School', ['Ocean Springs High School is an IB-certified public high school in Ocean Springs, Mississippi, United States.', ' The school serves students in grades 9–12 and is part of the Ocean Springs School District.']], ['Heber Springs School District', ['Heber Springs School District is a public school district based in Heber Springs, Arkansas, United States.', ' The Heber Springs School District provides early childhood, elementary and secondary education for more than 1,700 kindergarten through grade 12 students at its three facilities within Cleburne County, Arkansas.', ' The district is accredited by the Arkansas Department of Education (ADE).']], ['Ocean Springs, Mississippi', ['Ocean Springs is a city in Jackson County, Mississippi, United States, approximately 2 mi east of Biloxi and west of Gautier.', ' It is part of the Pascagoula, Mississippi Metropolitan Statistical Area.', ' The population was 17,225 at the 2000 U.S. Census.', ' As of the 2010 U.S. Census, the city of Ocean Springs had a population of 17,442.']], ['Rivercrest High School (Arkansas)', ['Rivercrest High School is a comprehensive public high school located in unincorporated Mississippi County, Arkansas, United States, in proximity to Marie and with a Wilson postal address.', \" It is the only high school administered by the Rivercrest School District (formerly the Southern Mississippi County School District) and houses the district's administrative offices.\"]], ['Willow Springs School District 108', ['Willow Springs School District 108 is a school district headquartered in Willow Springs, Illinois, in the Chicago metropolitan area.', ' It serves Willow Springs and unincorporated areas with Justice, Illinois addresses.', ' It has a single school, Willow Springs School, which was initially located in a school building with four rooms.', ' It began occupying its current site in the 1920s and the school building received an addition on its west side in 2005; this addition added main offices, computer labs, and a learning resource center.']], ['Hot Springs School District', ['Hot Springs School District is a public school district based in Hot Springs, Arkansas, United States.', ' The Hot Springs School District encompasses 33.15 mi2 of land including all or portions of Garland County communities including Hot Springs, Hot Springs National Park, Piney, and Lake Hamilton.']], ['Ocean Springs School District', ['The Ocean Springs School District is a public school district based in Ocean Springs, Mississippi (USA).']], ['Bermudian Springs School District', ['The Bermudian Springs School District is a small, rural, public school district created in 1970.', ' Bermudian Springs School District encompasses approximately 75 sqmi .', ' The district includes: the Boroughs of East Berlin and York Springs, as well as, the village of Idaville, Huntington Township, Latimore Township, Reading Township and a small part of Hamilton Township.', ' The 1990 U.S. census totals showed these communities have over 11,500 inhabitants.', ' According to 2007 local census data, it served a resident population of 13,077.', \" By 2010, the District's population had risen to 13,115 people.\", ' The educational attainment levels for the Bermudian Springs School District population (25 years old and over) were 83.9% high school graduates and 16.2% college graduates.']], ['Rivercrest School District', ['Rivercrest School District, formerly Southern Mississippi County School District, is a public school district based in Rivercrest High School in unincorporated Mississippi County, Arkansas, United States, in proximity to Marie and with a Wilson postal address.', ' The school district provides early childhood, elementary and secondary education for more than 1,300 prekindergarten through grade 12 students and employs more than 220 staff (including faculty) at its two facilities.', ' The district encompasses 363.77 mi2 of land in Mississippi County.']], ['Blue Springs R-IV School District', ['The Blue Springs R-IV School District is a school district that serves Blue Springs, Missouri in the Kansas City metropolitan area.', ' The district has an enrollment of over 13,000 students.', ' The mission statement of the Blue Springs R-IV School District is to create an educational community in which each individual acquires knowledge, develops skills, and functions as a literate citizen to achieve personal goals.', ' The Department of Elementary and Secondary Education reported that Blue Springs School District once again received a perfect score on the Annual Performance Report in 2011.', ' This is the eleventh year in a row that the district has received a perfect score.', ' This is determined by a number of factors including student achievement.', ' The Blue Springs School District is one of only seven school districts in this state to have eleven consecutive years of Distinction.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.770\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab9d7e4554299232ef4a21f', 'answer': 'constant support from propaganda campaigns', 'question': 'What was need in order to diverge from the chinese ritual uniting spouses by pre-arrangement between families?', 'supporting_facts': [['New Marriage Law', 1], ['Chinese marriage', 0]], 'context': [['Chinese Rites controversy', ['The Chinese Rites controversy was a dispute among Roman Catholic missionaries over the religiosity of Confucianism and Chinese rituals during the 17th and 18th centuries.', ' The debate centered over whether Chinese ritual practices of honoring family ancestors and other formal Confucian and Chinese imperial rites qualified as religious rites and were thus incompatible with Catholic belief.', ' The Jesuits argued that these Chinese rites were secular rituals that were compatible with Christianity, within certain limits, and should thus be tolerated.', ' The Dominicans and Franciscans, however, disagreed and reported the issue to Rome.']], ['Gui (vessel)', ['A gui is a type of bowl-shaped ancient Chinese ritual bronze vessel used to hold offerings of food, probably mainly grain, for ancestral tombs.', ' As with other shapes, the ritual bronzes followed early pottery versions for domestic use, and were recalled in later art in both metal, pottery, and sometimes stone.', ' The shape changed somewhat over the centuries but constant characteristics are a circular form (seen from above), with a rounded, wide, profile or shape from the side, standing on a narrower rim or foot.', ' There are usually two, or sometimes four, handles, and there may be a cover or a square base (or both).']], ['Fangxiangshi', ['The fangxiangshi 方相氏 was a Chinese ritual exorcist, the meaning of whose name is obscure but has been translated as \"one who sees in all (four) directions\", \"he who scrutinizes for evil in many directions\", and \"one who orients unwanted spirits in the direction to which they belong\".', ' Ancient Chinese texts record that he wore a bearskin with four golden eyes, and carried a lance and shield to expel malevolent spirits.', ' His primary duties were orchestrating the seasonal Nuo ritual to chase out disease-causing demons from houses and buildings, and leading a funeral procession to exorcize corpse-eating \"wangliang\" spirits away from a burial chamber.', ' From the Han dynasty through the Tang dynasty (3rd century BCE to 10th century CE), \"fangxiangshi\" were official \"wu\"-shaman specialists in the imperially sanctioned Chinese state religion; after the Tang, they were adapted into popular folk religion and symbolized by wearing a four-eyed mask.']], ['Chinese marriage', ['Traditional Chinese marriage (), as opposed to marriage in modern China, is a ceremonial ritual within Chinese societies that involve a union between spouses, sometimes established by pre-arrangement between families.', ' Within Chinese culture, romantic love and monogamy was the norm for most citizens.', \" Wedding rituals and customs often varied by region because of China's extensive and rich history and because of the numerous different cultures and ethno-linguistic groups that have been subsumed into modern Chinese culture.\"]], ['Chinese ritual mastery traditions', ['Chinese ritual mastery traditions, also referred to as ritual teachings (, sometimes rendered as \"Faism\"), or Folk Taoism (), or also Red Taoism (mostly in east China and Taiwan), constitute a large group of Chinese orders of ritual officers who operate within the Chinese folk religion but outside the institutions of official Taoism.', ' The \"masters of rites\", the \"fashi\" (法師), are also known in east China as \"hongtou daoshi\" (紅頭道士), meaning \"redhead\" or \"redhat\" \"daoshi\" (\"masters of the Tao\"), contrasting with the \"wutou daoshi\" (烏頭道士), \"blackhead\" or \"blackhat\" priests, of Zhengyi Taoism who were historically ordained by the Celestial Master.']], ['Ritual wine server (guang), Indianapolis', ['An elaborately decorated \"ritual wine server\" in the \"guang\" shape (; pinyin: gōng; Wade–Giles: kung) is a Chinese ritual bronze wine vessel, accession number 60.43, in the permanent Asian collection at the Indianapolis Museum of Art.', ' It dates to about 1100 BCE in the Shang dynasty period.', ' The piece is currently on display in the Arthur R. & Frances D. Baxter Gallery of the museum.']], ['Gu (vessel)', ['A gu is type of ancient Chinese ritual bronze vessel from the Shang and Zhou dynasties (i.e. 1600–256 BC).', ' It was used to drink wine or to offer ritual libations.']], ['Guang (vessel)', ['A guang or gong is a particular shape used in Chinese art for vessels, originally made as Chinese ritual bronzes in the Shang dynasty (c. 1600 – c. 1046 BC), and sometimes later in Chinese porcelain.', ' They are a type of ewer which was used for pouring rice wine at ritual banquets, and often deposited as grave goods in high-status burial.', ' Examples of the shape may be described as ewers, ritual wine vessels, wine pourers and similar terms, though all of these terms are also used of a number of other shapes, especially the smaller tripod \"jue\" and the larger \"zun\".']], ['New Marriage Law', [\"The New Marriage Law (also First Marriage Law, ) was a civil marriage law passed in the People's Republic of China on May 1, 1950.\", ' It was a radical change from existing patriarchal Chinese marriage traditions, and needed constant support from propaganda campaigns.', ' It has since been superseded by the Second Marriage Law of 1980.']], ['Ding (vessel)', ['Ding (鼎 ) were prehistoric and ancient Chinese cauldrons, standing upon legs with a lid and two facing handles.', ' They are one of the most important shapes used in Chinese ritual bronzes.', ' They were made in two shapes: round vessels with three legs and rectangular ones with four, the latter often called fanding.', ' They were used for cooking, storage, and ritual offerings to the gods or to ancestors.', ' The earliest recovered examples are pre-Shang ceramic ding at the Erlitou site but they are better known from the Bronze Age, particularly after the Zhou deemphasized the ritual use of wine practiced by the Shang kings.', ' Under the Zhou, the ding and the privilege to perform the associated rituals became symbols of authority.', \" The number of permitted ding varied according to one's rank in the Chinese nobility: the Nine Ding of the Zhou kings were a symbol of their rule over all China but were lost by the first emperor, Shi Huangdi in the late 3rd century\\u2009.\", ' Subsequently, imperial authority was represented by the Heirloom Seal of the Realm, carved out of the He Shi Bi jade; it was lost at some point during the Five Dynasties after the collapse of the Tang.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.771\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abb342055429966062416e8', 'answer': 'Captain Marvel', 'question': 'The world\\'s greatest Super-Heroes anthology showcased one of four superheroes known for speaking the phrase \"SHAZAM\", what was their name?', 'supporting_facts': [[\"The World's Greatest Super-Heroes\", 0], [\"The World's Greatest Super-Heroes\", 1], [\"The World's Greatest Super-Heroes\", 2], ['Captain Marvel (DC Comics)', 0], ['Captain Marvel (DC Comics)', 2], ['Captain Marvel (DC Comics)', 3]], 'context': [['Superman logo', ['The Superman shield, also known as the Superman logo, is the iconic emblem for the fictional DC Comics superhero Superman.', \" As a representation of one of the first superheroes, it served as a template for character design decades after Superman's first appearance.\", ' The tradition of wearing a representative symbol on the chest was mimicked by many subsequent superheroes, including Batman, Spider-Man, the Fantastic Four, Green Lantern, the Flash, Wonder Woman, Hawkman, and many others.']], [\"The World's Greatest Super-Heroes\", [\"The World's Greatest Super-Heroes is the name to the oversized slipcased hardcover anthology collection, that reprinted six oversized graphic novels all created by artist Alex Ross and writer Paul Dini.\", ' Four graphic novels were originally published to celebrate the 60th anniversary of DC main characters: Superman, Batman, Captain Marvel and Wonder Woman, thus \"\" were published in 1998, \"\" in 1999, \"\" in 2000 and finally \"\" in 2001.', ' Alex Ross original idea was make a statement about the four superhero comics archetypes: Science Fiction (Superman), Crime/Mystery (Batman), Magic (Captain Marvel) and Myth (Wonder Woman).']], ['Detroit Economic Club', ['The Detroit Economic Club, headquartered at 211 West Fort Street in downtown Detroit, Michigan, was formed in 1934 as a platform for the discussion and debate of important business, government and social issues.', \" It is known internationally as a top speaking forum for prominent business, academic, and government officials, who address members and their guests at the Club's 35 meeting season.\", ' With more than 3,500 members, the DEC is a forum for vital issues.', ' The DEC claims to have hosted every sitting U.S. President since Richard Nixon and be ranked among the top speaking platforms in the world.', \" The DEC claims to be one of the most valued podiums for CEO's in the world and one of the top five executive speaking forums in America.\"]], ['List of Jewish superheroes', ['There also exists a team of Judaicly themed superheroes known as \"The Jewish Hero Corps\", printed by Leviathan press.', ' They include Menorah Man, Yarmulke Youth, Matzah Woman, Driedel Maidel, Magen David, Minyan Man, and Shabbas Queen.', ' They are not commonly recognized as actual characters primarily because only one issue of the comic is known to have been produced at this time.']], ['Exposure (U.S. TV series)', ['Exposure is a short-film oriented science-fiction anthology series that aired on the Sci-Fi Channel between the years of 2000 and 2002.', \" The series showcased the short sci-fi films of both unknown and known (Tim Burton, George Lucas and Kevin Smith) filmmakers, giving rise to the channel's own Exposure Studios.\", ' Hosted by actress Lisa Marie, the films presented a wide range of science fiction subject matter.', ' The series received poor ratings and was canceled in the fall of 2002.', ' The series was originally shown on Sundays at 10:00pm EST and was repeated the following Saturday at 2:00am EST, later on the time was changed to 11:00pm EST and still repeated the following Saturday well after Midnight, which probably is a major reason for the poor ratings this show received.', ' The series also had two guest hosts.', ' Terry Farrell would host the \"Best of Season One\" episode and director Kevin Smith hosted the \"Star Wars Short Films Showcase\".']], ['Guo Hong', [\"Guo Hong (; nicknamed The Great Wall of China), is a goaltender for the China women's national ice hockey team.\", ' During many international contests, she has been known to block over 50 shots a game.', ' At the 2002 Winter Olympics, Hong registered a save percentage of 88.79 save percentage.', ' She led all goaltenders at the event in saves and shots against.', ' The 1996 Pacific Rim Tournament showcased one of the best games of her career.', ' In a game against the Canadian National Women’s Team, Hong stopped 38 of 39 shots in a 1-0 loss.', \" Hong has represented the China women's national ice hockey team for over ten years.\"]], ['Gold (Donna Summer album)', [\"Gold is one of Donna Summer's greatest hits compilations.\", ' Donna Summer\\'s entry in Universal Music\\'s two-disc compilation series \"Gold\" is more or less a re-release of 1993\\'s \"The Donna Summer Anthology\", with the most noticeable differences being the cover art and that \"Gold\" includes four of her 90\\'s Club and R&B hits, which came out after the \"Anthology\".', ' Also, other 80\\'s European hits, such as \"Dinner With Gershwin\", and the 7 inch remix of \"Love\\'s About to Change My Heart\", that were not included on the \"Anthology\", are present here.', ' However, the two tracks from 1981\\'s shelved Geffen Records album \"I\\'m a Rainbow\" on disc two are left out, as are \"Once Upon A Time\" and \"Rumour Has It\", both from the 1977 album \"Once Upon a Time\", the hit single \"Cold Love\" from 1980 album \"The Wanderer\", and the album track \"Friends Unknown\" from \"Mistaken Identity\".', ' As of August 10, 2006, the album sold 30,000 in United States, according to Nielsen Soundscan.']], ['Captain Marvel (DC Comics)', ['Captain Marvel, also known as Shazam ( ), is a fictional superhero appearing in American comic books published by DC Comics.', ' Artist C. C. Beck and writer Bill Parker created the character in 1939.', ' Captain Marvel first appeared in \"Whiz Comics\" #2 (cover-dated Feb. 1940), published by Fawcett Comics.', ' He is the alter ego of Billy Batson, a boy who, by speaking the magic word \"SHAZAM\" (acronym of six \"immortal elders\": Solomon, Hercules, Atlas, Zeus, Achilles, and Mercury), can transform himself into a costumed adult with the powers of superhuman strength, speed, flight, and other abilities.']], ['Legion of Super Heroes (TV series)', ['Legion of Super Heroes is an American animated television series produced by Warner Bros.', ' Animation that debuted on September 23, 2006, and is based on characters owned by DC Comics.', \" The series centers on a young Superman's adventures in the 31st century, fighting alongside a group of futuristic superheroes known as the Legion of Super-Heroes.\", ' The show was produced by its main designer James Tucker, a co-producer of the \"Justice League Unlimited\" series, for the Kids\\' WB line on The CW network.']], ['Four-Star Spectacular', ['Four-Star Spectacular was an anthology comic book series published by DC Comics in the mid-1970s.', ' The series was edited by E. Nelson Bridwell and ran for six issues from March/April 1976 to January/February 1977.', ' The books were in the \"giant size\" format and consisted mostly of superhero reprints, with some new material.', \" A total of four characters from DC's roster of superheroes appeared in each issue — hence the title.\", \" (Half of the title's issues, however, only featured three stories.)\", ' Each issue featured a Superboy story, a Wonder Woman story, and at least one other story (usually a team-up story).', ' All issues featured cover art by DC artist Ernie Chua.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.772\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8727dd5542991e771816f8', 'answer': 'Florida Panthers', 'question': 'Wojtek Wolski played for what team based in the Miami metropolitan area?', 'supporting_facts': [['Wojtek Wolski', 1], ['Florida Panthers', 0]], 'context': [['List of Miami Dolphins starting quarterbacks', ['The Miami Dolphins are a professional American football team based in the Miami metropolitan area.', ' They are members of the East Division of the American Football Conference (AFC) in the National Football League (NFL).', ' Lawyer Joe Robbie and actor Danny Thomas were granted enfranchisement on August 15, 1965, committing their team as the ninth member of the American Football League (AFL).']], ['Sport in Miami', ['The city of Miami and the Miami metropolitan area are home to four major league sports teams — the Miami Dolphins of the National Football League, the Miami Heat of the National Basketball Association, the Miami Marlins of Major League Baseball, and the Florida Panthers of the National Hockey League.', ' As well as having all five major professional teams, Miami is also the future home to the Major League Soccer expansion team led by David Beckham.']], ['Miami Kickers', [\"The Miami Kickers were an American women's soccer team based in the Miami metropolitan area.\", \" Founded in 2005, the team played in the Women's Premier Soccer League (WPSL) from 2006–2010.\", ' They played their home games in the stadium on the campus of American Heritage School in Plantation, Florida, 26 miles north of downtown Miami.']], ['Florida Panthers', ['The Florida Panthers are a professional ice hockey team based in the Miami metropolitan area.', ' They are members of the Atlantic Division of the Eastern Conference of the National Hockey League (NHL).', ' It was founded in 1993 as an expansion team.', ' They play home games at the BB&T Center in Sunrise, Florida; the Panthers are the southernmost team in the NHL.']], ['Tidal flooding', ['Tidal flooding, also known as sunny day flooding or nuisance flooding, is the temporary inundation of low-lying areas, especially streets, during exceptionally high tide events, such as at full and new moons.', ' The highest tides of the year may be known as the king tide, with the month varying by location.', ' In Florida, controversy was created when state-level government mandated that the term \"nuisance flooding\" and other terms be used in place of terms such as sea level rise, climate change and global warming, prompting allegations of climate change denial, specifically against Governor Rick Scott.', ' This amid Florida, specifically South Florida and the Miami metropolitan area being one of the most at risk areas in the world for the potential effects of sea level rise, and where the frequency and severity of tidal flooding events increased in the 21st century.', ' The issue is more bipartisan in South Florida, particularly in places like Miami Beach, where a several hundred million dollar project is underway to install more than 50 pumps and physically raise roads to combat the flooding, mainly along the west side of South Beach, formerly a mangrove wetland where the average elevation is less than one meter (3.3 feet).', ' In the Miami area, where the vast majority of the land is below 10 ft , even a one-foot increase over the average high tide can cause widespread flooding.', ' The 2015 and 2016 king tide event levels reached about 4 ft MLLW, 3 ft above mean sea level, or about 2 ft NAVD88, and nearly the same above MHHW.', ' While the tide range is very small in Miami, averaging about 2 ft , with the greatest range being less than 2 m , the area is very acute to minute differences down to single inches due to the vast area at low elevation.', ' NOAA tide gauge data for most stations shows current water level graphs relative to a fixed datum, as well as mean sea level trends for some stations.', ' During the king tides, the local Miami area tide gauge at Virginia Key shows levels running at times 1 ft or more over datum.']], ['Miami metropolitan area', ['The Miami metropolitan area, also known as the Greater Miami Area or South Florida, is the 67th largest metropolitan area in the world and the eighth-largest metropolitan area in the United States.', ' It is entirely located in the southern portion of the U.S. state of Florida.', ' With 6,066,387 inhabitants as of 2016, the Miami metropolitan area is the most populous in Florida and second largest in the Southeastern United States.']], ['West Palm Beach, Florida', ['West Palm Beach is a city in and the county seat of Palm Beach County, Florida, United States.', ' It is one of the three main cities in South Florida.', ' The population was 100,343 (revised) at the 2010 census.', ' The University of Florida Bureau of Economic and Business Research (BEBR) estimates a 2016 population of 108,896, a 7.9% increase from 2010.', ' It is the oldest municipality in the Miami metropolitan area, having been incorporated as a city two years before Miami in November 1894.', ' Although West Palm Beach is located approximately 68 mi north of Downtown Miami, it is still considered a principal city within the Miami metropolitan area, due to the solid urbanization between both cities.', ' The estimated population of the Miami metropolitan area, which includes all of Palm Beach County, was 6,012,331 people at the 2015 census.']], ['Wojtek Wolski', ['Wojciech \"Wojtek\" Wolski (] ; born February 24, 1986) is a Polish-Canadian professional ice hockey left winger currently playing for HC Kunlun Red Star of the Kontinental Hockey League (KHL).', ' In the NHL, he has played for the Colorado Avalanche, Phoenix Coyotes, New York Rangers, Florida Panthers, and the Washington Capitals.', ' During the 2012 NHL lockout, he played for Ciarko PBS Bank KH Sanok in the PHL, the top-tier hockey league in Poland.', ' While he holds dual citizenship, Wolski is a product of the Canadian training system and is currently ineligible to represent Poland internationally.']], ['Florida Bobcats', ['The Florida Bobcats were an Arena Football League (AFL) team based in Sunrise, Florida.', ' They were previously known as the Sacramento Attack and the Miami Hooters, and played in the AFL for a total of ten seasons, the last seven in West Palm Beach and Sunrise in the Miami metropolitan area.']], ['Sunrise, Florida', ['Sunrise is a city in central-western Broward County, Florida, United States, in the Miami metropolitan area.', ' It was incorporated in 1961 by Norman Johnson – a developer whose Upside-Down House attracted buyers to what was then a remote area.', ' As of the 2010 census, the city had a total population of 84,439.', ' It is a principal city of the Miami metropolitan area, which was home to an estimated 6,012,331 people at the 2015 census.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.774\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7622d75542994ccc9186f1', 'answer': '\"Leni\" Riefenstahl', 'question': 'Who was born earlier, Garry Marshall or Leni Riefenstahl?', 'supporting_facts': [['Garry Marshall', 0], ['Leni Riefenstahl', 0]], 'context': [['Garry Marshall', ['Garry Kent Marshall (November 13, 1934 – July 19, 2016) was an American actor, director, producer, writer, and voice artist best known for creating \"Happy Days\" and its various spin-offs, developing Neil Simon\\'s 1965 play \"The Odd Couple\" for television, and directing \"Pretty Woman\", \"Runaway Bride\", \"Valentine\\'s Day\", \"New Year\\'s Eve\", \"Mother\\'s Day\", \"\"The Princess Diaries\", and \"\".', ' He provided the voice of Buck Cluck in \"Chicken Little\".']], [\"Leni Riefenstahl's Memoiren\", [\"Leni Riefenstahl's Memoiren is the 1987 autobiography of German film director, Leni Riefenstahl.\", ' The book received a 1993 American release and coincided with the release of the acclaimed documentary; \"The Wonderful, Horrible Life of Leni Riefenstahl\" as well as Riefenstahl\\'s ninetieth birthday.', ' It was featured on the 1993 \"New York Times\" list of notable books of the year.']], ['The White Ecstasy (film)', ['The White Ecstasy (German: Der weisse Rausch) is a 1931 German mountain film written and directed by Arnold Fanck and starring Hannes Schneider, Leni Riefenstahl, Guzzi Lantschner, and Walter Riml.', ' The film is about the skiing exploits of a young village girl (played by Riefenstahl), and her attempts to master the sport of skiing and ski-jumping aided by the local ski expert (played by Schneider).', ' Filmed on location in Sankt Anton am Arlberg, the film was one of the first to use and develop outdoor film-making techniques and featured several innovative action-skiing scenes.']], ['The Blue Light (1932 film)', ['The Blue Light (German: \"Das blaue Licht\") is a black-and-white 1932 film written and directed by Leni Riefenstahl and Béla Balázs, with uncredited scripting by Carl Mayer.', \" In Riefenstahl's film version, the witch, Junta, played by Riefenstahl, is intended to be a sympathetic character.\", ' Filming took place in the Brenta Dolomites, in Ticino, Switzerland, and Sarntal, Italy.']], ['The Holy Mountain (1926 film)', ['The Holy Mountain (German: \"Der heilige Berg\" ) is a 1926 German mountain film directed by Arnold Fanck and starring Leni Riefenstahl, Luis Trenker and Frida Richard.', \" It was the future filmmaker Riefenstahl's first screen appearance as an actress.\", ' Written by Arnold Fanck and Hans Schneeberger, the film is about a dancer who meets and falls in love with an engineer at his cottage in the mountains.', ' After she gives her scarf to one of his friends, the infatuated friend mistakenly believes that she loves him.', ' When the engineer sees her innocently comforting his friend, he mistakenly believes she is betraying him.']], ['Charlotte Riefenstahl', ['Charlotte Riefenstahl (24 May 1899 in Bielefeld, Germany – 6 January 1993 in Northfield, Minnesota, United States ) was a German physicist.', ' She has no relation to Leni Riefenstahl, the notable German filmmaker.']], ['Leni Riefenstahl', ['Helene Bertha Amalie \"Leni\" Riefenstahl (] ; 22 August 1902 – 8 September 2003) was a German film director, producer, screenwriter, editor, photographer, actress and dancer.']], ['Sepp Allgeier', ['Josef “Sepp” Allgeier (6 February 1895 – 11 March 1968) was a German cinematographer who worked on around fifty features, documentaries and short films.', ' He began his career as a cameraman in 1911 for the Expreß Film Co. of Freiburg.', ' In 1913 he filmed newsreels in the Balkans.', ' He then became an assistant to Arnold Fanck, a leading director of Mountain films.', ' He worked frequently with Luis Trenker and Leni Riefenstahl, both closely associated with the genre.', ' He was Riefenstahl\\'s lead cameraman on her 1935 propaganda film \"Triumph of the Will\".', ' During the Second World War, Allgeier filmed material for newsreels.', ' He later worked in West German television.', ' His son is the cinematographer Hans-Jörg Allgeier.']], ['Leni Riefenstahl: Her Dream of Africa', ['Leni Riefenstahl: Her Dream of Africa (\"Leni Riefenstahl: Ihr Traum von Afrika\") is a 2000 documentary-film by Ray Müller.', ' The film follows Leni Riefenstahl\\'s return to Sudan to visit the Nuba tribe whom she published photographs of in best-sellers such as \"The Last of the Nuba\" and \"The People of Kau\".', ' It is the second collaboration between Riefenstahl and Müller.', ' She was the subject of his acclaimed 1993 documentary \"The Wonderful, Horrible Life of Leni Riefenstahl\", which followed her life and reflected on her Nuba activities.']], ['The Wonderful Horrible Life of Leni Riefenstahl', ['The Wonderful, Horrible Life of Leni Riefenstahl (German: \"Die Macht der Bilder: Leni Riefenstahl\" ) is a 1993 German documentary film about the life of German film director Leni Riefenstahl, directed by Ray Müller.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.774\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a80be9655429938b61421b5', 'answer': 'Fred Willard', 'question': 'In addition to Jennifer Tilly, Brian Backer, Sally Kellerman, Nedra Volz, Clara Peller, Wendie Jo Sperber and the actor born on June 22 1958 in Wilmette, Illinois, who else starred in Moving Violations?', 'supporting_facts': [['Moving Violations', 0], ['John Murray (actor)', 0]], 'context': [['Jennifer Tilly', ['Jennifer Tilly (born Jennifer Ellen Chan; September 16, 1958) is an American-Canadian actress and poker player.', \" She is a World Series of Poker Ladies' Event bracelet winner.\", ' She was nominated for an Academy Award for Best Supporting Actress for her role as Olive Neal in the film \"Bullets over Broadway\" (1994).', ' Her other film roles include \"Let It Ride\" (1989) \"Made in America\" (1993), \"Bound\" (1996), \"Liar Liar\" (1997) and \"Bride of Chucky\" (1998).', ' She has done extensive voice-over work including Celia in \"Monsters, Inc.\" (2001).', ' She is the older sister of actress Meg Tilly.']], ['Dempsey (film)', ['Dempsey is a 1983 TV movie based on the life of the heavyweight boxer Jack Dempsey that starred Treat Williams and Sally Kellerman.']], ['Wendie Jo Sperber', ['Wendie Jo Sperber (September 15, 1958 – November 29, 2005) was an American actress, known for her performances in the films \"I Wanna Hold Your Hand\" (1978), \"Bachelor Party\" (1984), and \"Back to the Future\" (1985) and as well as the television sitcoms \"Bosom Buddies\" (1980–1982) and \"Private Benjamin\" (1982–1983).']], ['Nedra Volz', ['Nedra Volz (née Gordonier, June 18, 1908 – January 20, 2003) was an American actress.']], ['John Murray (actor)', ['John Murray (born June 22, 1958 in Wilmette, Illinois) is an American actor, writer and producer.']], ['Clara Peller', ['Clara Peller (August 4, 1902 – August 11, 1987), was a manicurist and American character actress who, at the age of 81, starred in the 1984 \"Where\\'s the beef?', '\" advertising campaign for the Wendy\\'s fast food restaurant chain, created by the Dancer Fitzgerald Sample advertising agency.']], ['Brian Backer', ['Brian Backer (born December 5, 1956) is an American actor who has starred in film and on television.', ' He is best known for his role in the 1982 hit comedy film \"Fast Times at Ridgemont High\" as shy teenager Mark \"Rat\" Ratner.', ' His other films include the 1985 comedy film \"Moving Violations\" (playing the role of Scott Greeber) and the 1987 comedy film \"\".']], ['Delgo', ['Delgo is a 2008 American computer-animated adventure romantic comedy fantasy film directed by Marc F. Adler and Jason Maurer, written by Scott Biear, Patrick J. Cowan, Carl Dream and Jennifer A. Jones.', ' It stars Freddie Prinze, Jr., Jennifer Love Hewitt, Anne Bancroft, Chris Kattan, Louis Gossett Jr., Val Kilmer and Malcolm McDowell with narration by Sally Kellerman.', ' It was distributed by Freestyle Releasing with music by Geoff Zanelli and produced by Electric Eye Entertainment Corporation and Fathom Studios, a division of Macquarium Intelligent Communications, which began development of the project in 1999.']], ['Sally Schoch', ['Sally Schoch (born 1934) MFA is an American artist and abstract painter living in Wilmette, Illinois.', ' She received her Master of Fine Arts degree from the School of the Art Institute of Chicago.', ' Schoch has worked in fiber arts and painted in oils and watercolor, and is most known for her abstract paintings of flowers.', \" Schoch has received commissions for works by Marshall Field's, Bank of America, and other organizations.\", \" She is a member of the Wilmette Art Guild and the Chicago Artists' Coalition.\", ' Her work has been exhibited primarily in the Midwestern United States']], ['Moving Violations', ['Moving Violations is a 1985 comedy film starring John Murray, Jennifer Tilly, Brian Backer, Sally Kellerman, Nedra Volz, Clara Peller, Wendie Jo Sperber and Fred Willard.', ' It was directed by Neal Israel and was the film debut of Don Cheadle.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.774\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a79b525554299029c4b5f67', 'answer': 'traditional music', 'question': 'What performing traditionally from the longest continental mountain range in the world, are Grupo Aymara famous for?', 'supporting_facts': [['Grupo Aymara', 0], ['Andes', 0]], 'context': [['Aquichua', ['Aquichua (possibly from Aymara, \"jaqhi\" precipice, cliff, Aymara and Quechua \"chuwa\" plate, \"cliff plate\") is a mountain in the Vilcanota mountain range in the Andes of Peru, about 5300 m high.', ' It is located in the Cusco Region, Quispicanchi Province, Marcapata District.', ' Aquichua is situated north east of the lake Sibinacocha and the mountain Chumpe and north of the Yayamari.']], ['Apachita Pura Pura', ['Apachita Pura Pura (Aymara and Quechua \"apachita\" the place of transit of an important pass in the principal routes of the Andes; name in the Andes for a stone cairn, a little pile of rocks built along the trail in the high mountains, Aymara \"pura pura\" \"Xenophyllum\" (or a species of it), also spelled \"Apacheta Pura Pura\") is a mountain in the Apolobamba mountain range in Bolivia, about 5360 m high.', \" It is situated in the La Paz Department, Franz Tamayo Province, Pelechuco Municipality, northwest of the mountain Rit'i Apachita and southeast of Chuquyu.\"]], ['Alcamarinayoc', ['Alcamarinayoc (possibly from Aymara and Quechua \"allqamari\" mountain caracara) or Colque Cruz (possibly from Aymara and Quechua \"qullqi\" money, silver, Spanish \"cruz\" cross) is a 6102 m mountain in the Vilcanota mountain range in the Andes of Peru.', ' It is situated in the Cusco Region, Quispicanchi Province, Ocongate District.', ' Alcamarinayoc lies northwest of the peak of Chumpe, north of Quevesere and northeast of Ichhu Ananta.']], ['Wind River Range', ['The Wind River Range (or \"Winds\" for short), is a mountain range of the Rocky Mountains in western Wyoming in the United States.', ' The range runs roughly NW-SE for approximately 100\\xa0miles (161\\xa0km).', ' The Continental Divide follows the crest of the range and includes Gannett Peak, which at 13,804\\xa0feet (4,207\\xa0m), is the highest peak in Wyoming.', ' There are more than 40 other named peaks in excess of 13,000\\xa0feet (3,962\\xa0m).', ' With the exception of the Grand Teton in the Teton Range, the next 19 highest peaks in Wyoming after Gannett are also in the Winds.', ' Two large National Forests including three wilderness areas encompass most of the mountain range.', ' Shoshone National Forest is on the eastern side of the continental divide while Bridger-Teton National Forest is on the west.', ' Both National Forests and the entire mountain range are an integral part of the Greater Yellowstone Ecosystem.', ' Portions of the range are also inside the Wind River Indian Reservation.']], ['Kunturi (Condesuyos)', ['Kunturi (Aymara for condor, hispanicized spelling \"Condori\") is a mountain in the Wansu mountain range in the Andes of Peru, about 5208 m high.', ' It is situated in the Arequipa Region, Condesuyos Province, Cayarani District, and in the La Unión Province, Puyca District, northeast of the mountain Hatunpata \"(Atunpata)\".', ' Kunturi lies south of the river Uqururu (Aymara and Quechua for \"Mimulus glabratus\", hispanicized \"Ojoruro\"), also known as Sumana or Cotahuasi, which flows to the Cotahuasi Canyon in the southwest.']], ['Andes', ['The Andes or Andean Mountains (Spanish: \"Cordillera de los Andes\" ) are the longest continental mountain range in the world.', ' They form a continuous highland along the western edge of South America.', ' This range is about 7000 km long, about 200 to wide (widest between 18° south and 20° south latitude), and of an average height of about 4000 m .', ' The Andes extend from north to south through seven South American countries: Venezuela, Colombia, Ecuador, Peru, Bolivia, Argentina and Chile.']], ['Kunturi (Ikmaqucha)', ['Kunturi (Aymara for condor, hispanicized spelling \"Condori\") is a mountain in the Wansu mountain range in the Andes of Peru, about 5000 m high.', ' It is located in the Arequipa Region, La Unión Province, Puyca District.', ' Kunturi lies east of a lake named Ikmaqucha.', \" Taypi Q'awa is the mountain northeast of it.\", ' The intermittent streams south of Kunturi flow to the Uqururu (Aymara and Quechua for \"Mimulus glabratus\", hispanicized \"Ojoruro\"), also known as Sumana or Cotahuasi, which flows to the Cotahuasi Canyon in the southwest.']], ['Huillolluni', ['Huillolluni (possibly from Aymara and Quechua \"willullu\" poor / orphan, Aymara \"-ni\" a suffix to indicate ownership, \"the one with an orphan\") is a mountain in the Vilcanota mountain range in the Andes of Peru, about 5000 m high.', ' It is situated in the Cusco Region, Quispicanchi Province, Marcapata District, and in the Paucartambo Province, Kosñipata District.', ' Huillolluni lies north-east of the mountain Qullqipunku and north-west of the mountain Ancahuachana.']], ['Grupo Aymara', ['Grupo Aymara are a Bolivian folk troupe that have been acclaimed worldwide for its inspiring interpretations of traditional music of pre-Hispanic and contemporary music of the Andes, particularly that of the Aymara and Quechua speaking people of Bolivia.', ' They perform their evocative music on indigenous flutes, panpipes and drums, as well as stringed instruments introduced since the Spanish conquest.']], ['Cajamarca Region', ['Cajamarca (] ; Quechua: \"Kashamarka\" ; Aymara: \"Qajamarka\" ) is a region in Peru.', ' The capital is the city of Cajamarca.', ' It is located in the north part of the country and shares a border with Ecuador.', ' It is located at heights reaching 2700 m above sea level in the Andes Mountain Range, the longest mountain range in the world.', ' Part of its territory includes the Amazon Rainforest, in total the largest in the world.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.775\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a88696b554299206df2b25b', 'answer': 'musician', 'question': 'What type of profession does Chris Jericho and Gary Barlow have in common?', 'supporting_facts': [['Chris Jericho', 0], ['Gary Barlow', 0]], 'context': [['Music Music Music', ['Music Music Music is a 2008 album by John Barrowman preceded by the digital-only single \"What About Us?\"', ', written by Gary Barlow and Chris Braide.', ' The album consists mostly of cover versions and was produced by Simon Franglen (who has worked with Celine Dion, Whitney Houston, Barbra Streisand, The Ten Tenors) and Graham Stack (who has worked with Kylie Minogue, Tina Turner, Rod Stewart), with the exception of the first single \"What About Us?\"', ' (UK #122) which was produced by Chris Braide.', ' The album debuted and peaked at #35 in the UK, making it Barrowman\\'s second top forty album, after the #22 peak of \"Another Side\" in 2007.', ' The set also produced his first UK Top 75 charting single, \"I Made It Through the Rain\", which spent one week in the charts at #14 after being championed by Chris Moyles in August 2009.']], ['Gary Barlow', ['Gary Barlow OBE (born 20 January 1971) is an English singer, songwriter and record producer.', ' He is frontman and lead vocalist of British pop-group Take That.', ' Barlow also served as head judge of \"The X Factor UK\" from 2011 to 2013 and \"Let It Shine\" in 2017.', \" Barlow is one of Britain's most successful songwriters, having written fourteen number one singles and twenty-four top 10 hits.\", ' He has had three number one singles, six top 10 singles and two number one albums as a solo artist, and has had seventeen top 5 hits, twelve number one singles and seven number one albums with Take That.', ' He is also a six-time recipient of the Ivor Novello Award and has sold over 50 million records worldwide.']], ['Gary Barlow: In Concert', ['Gary Barlow: In Concert was the first full solo tour that Gary Barlow had performed in over 13 years.', ' Leg 1 saw him performing songs from his \"incredible music career spanning over 20 years\" in front of a sell out audience, whilst also raising money for The Prince\\'s Trust and The Foundation of Prince William and Prince Harry during two nights at the Royal Albert Hall.', ' It was announced on 15 October 2012 that Barlow would go on a full solo tour for his second leg of shows, lasting two months around the UK and Ireland.']], ['Fozzy', ['Fozzy is an American heavy metal band formed in Atlanta, Georgia, in 1999 by lead singer Chris Jericho and guitarist Rich Ward.', ' The band is currently signed to Century Media Records and has released two studio albums through this label.', \" The band's current lineup consists of Chris Jericho, Rich Ward, Frank Fontsere, Billy Grey and Paul Di Leo.\", ' Jericho has characterized the band by saying, \"If Metallica and Journey had a bastard child, it would be Fozzy.\"', ' As of July 2014, the band has released six studio albums and one live album.']], ['Chris Jericho', ['Christopher Keith Irvine (born November 9, 1970), better known by the ring name Chris Jericho, is a Canadian-American professional wrestler, musician, media personality, actor, author, podcaster, and businessman signed to WWE on the SmackDown brand, where he is currently on a hiatus due to an expected tour with Fozzy.', ' He is known for his over-the-top, rockstar persona.']], [\"Gary Barlow: On Her Majesty's Service\", [\"Gary Barlow: On Her Majesty's Service is a documentary featuring the process of Gary Barlow writing the Queen's Diamond Jubilee official single and travelling across The Commonwealth enlisting the help of singers and musicians to feature in the track.\"]], ['List of songs recorded by Gary Barlow', ['The English singer and songwriter Gary Barlow has recorded songs for three solo studio albums, a joint album and has also collaborated with other artists on their respective singles.', ' In 1996, Take That, a boy band consisted of five members, including Barlow, disbanded after 6 years recording music together.', ' Three months after the release of their then-final single, the singer launched his solo debut song, \"Forever Love\"; it was written solely by Barlow and featured more mature sound than the work previously released by Take That.', ' The single has been succeeded by Barlow\\'s 1997 debut studio album, \"Open Road\", a primary pop and blue-eyed soul oriented record.', ' It was mainly written and composed by Barlow himself, however, he listed some songwriters in the penning process.', ' American entertainer Madonna and producer Shep Pettibone wrote \"Love Won\\'t Wait\" for the album, although it was originally planned for a Madonna project which she later abandoned.', ' Barlow worked with American songwriter Dianne Warren with whom he co-wrote the song \"My Commitment\" for the album.', ' Howard Perdew and Andy Spooner wrote \"So Help Me Girl\", a single for which Barlow recorded a Spanish language version titled \"Ayúdame\" (English: \"Help Me\").']], ['Children in Need Rocks the Royal Albert Hall', ['Children in Need Rocks the Royal Albert Hall was a charity music concert held at the Royal Albert Hall in London, England on 12 November 2009.', ' The concert was organised by Take That singer-songwriter Gary Barlow as one of a series of events to raise money for Children in Need 2009.', ' The huge success of the concert inspired Barlow to organise \"Children in Need Rocks Manchester\" at the Manchester Arena, Manchester in 2011.']], ['Children in Need Rocks Manchester', ['Children in Need Rocks Manchester was a charity music concert held at the Manchester Arena in Manchester, England, on 17 November 2011.', ' The concert was organised by Take That singer-songwriter and \"The X Factor\" judge Gary Barlow as one of a series of events to raise money for Children in Need 2011.', ' It became the second \"Children in Need Rocks\" concert organised by Barlow, after the \"Children in Need Rocks the Royal Albert Hall\" in 2009.']], ['Let Me Go (Gary Barlow song)', ['\"Let Me Go\" is a song by British singer-songwriter Gary Barlow.', ' It was released in Ireland on 15 November 2013 and in the United Kingdom on 17 November 2013 as the lead single from his fourth solo album, \"Since I Saw You Last\" (2013).', ' It was written by Barlow and produced by Steve Power.', ' \"Let Me Go\" peaked at number two in the UK Singles Chart, becoming Barlow\\'s sixth solo top 10 hit in the UK.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.776\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a735c3a55429901807daff7', 'answer': 'mathematician', 'question': 'Nikolay Mitrofanovich Krylov and Anatoly Fomenko both held what academic title?', 'supporting_facts': [['Nikolay Mitrofanovich Krylov', 0], ['Anatoly Fomenko', 0]], 'context': [['Nicolai V. Krylov', ['Nicolai Vladimirovich Krylov (Russian: Никола́й Влади́мирович Крыло́в ; born 5 June 1941) is a Russian mathematician specializing in partial differential equations, particularly stochastic partial differential equations and diffusion processes.', ' Krylov studied at Lomonosov University, where he in 1966 under E. B. Dynkin attained a doctoral candidate title (similar to a PhD) and in 1973 a Russian doctoral degree (somewhat more prestigious than a PhD).', ' He taught from 1966 to 1990 at the Lomonosov University and is since 1990 a professor at the University of Minnesota.', ' At the beginning of his career (starting from 1963) he, in collaboration with Dynkin, worked on nonlinear stochastic control theory, making advances in the study of convex, nonlinear partial equations of 2nd order (\"i.e.\" Bellman equations), which were examined with stochastic methods.', ' This led to the Evans-Krylov theory, for which he received with Lawrence C. Evans in 2004 the Leroy P. Steele Prize of the American Mathematical Society (for work done simultaneously and independently by both Krylov and Evans).', ' They proved the second order differentiability (Hölder continuity of the second derivative) of the solutions of convex, completely nonlinear, second order elliptical partial differential equations and thus the existence of \"classical solutions\" (Theorem of Evans-Krylov).', ' He was in 1978 at Helsinki and in 1986 at Berkeley an Invited Speaker for the ICM.', ' He received the Humboldt Research Award in 2001.', ' In 1993 he was elected a member of the American Academy of Arts and Sciences (1993).', ' He should not be confused with the mathematician Nikolay M. Krylov.']], ['Anatoly Fomenko', ['Anatoly Timofeevich Fomenko (Russian: Анато́лий Тимофе́евич Фоме́нко ) (born 13 March 1945 in Stalino, USSR) is a Soviet and Russian mathematician, professor at Moscow State University, well known as a topologist, and a member of the Russian Academy of Sciences.', ' He is author of a pseudoscientific theory known as New Chronology.', ' He is also a member of the Russian Academy of Natural Sciences (1991).']], ['Doctor of Business Administration', ['The Doctor of Business Administration (abbreviated DBA, D.B.A., DrBA, or Dr.B.A.)', ' is a research doctorate awarded on the basis of advanced study and research in the field of business administration.', ' Along with research skills the doctorate focuses on business intelligence and original theoretical study.', ' The D.B.A. is a terminal degree in business administration, and is equivalent to the Ph.D in Business Administration.', ' Along with the Ph.D, it represents the highest academic qualification in business administration.', ' Successful completion of a D.B.A. or Ph.D in Business Administration is required to gain employment as a full-time, tenure-track university professor or postdoctoral researcher in the field.', ' As with other earned research doctorates, individuals with the degree are awarded the academic title doctor, which is often represented via the English honorific \"Dr.\" or the post-nominal letters \"D.B.A.\", \"DBA\", \"Dr.B.A.\"', ', or \"DrBA\".']], ['Associate professor', ['Associate professor (frequently capitalized as Associate Professor) is an academic title that can have different meanings.', ' In North America and universities elsewhere using the North American system, it is a position between assistant professor and a full professorship.', ' In some Commonwealth countries, the title associate professor is often used in place of reader, which is used in the United Kingdom and a number of other Commonwealth universities; this usage is typical of universities in Australia and New Zealand, as well as in South Africa, India, parts of Southeast Asia, Ireland and other countries.', ' The title associate professor in those countries, like the title reader, corresponds to a full professorship in North America.']], ['Describing function', ['In control systems theory, the describing function (DF) method, developed by Nikolay Mitrofanovich Krylov and Nikolay Bogoliubov in the 1930s, and extended by Ralph Kochenburger is an approximate procedure for analyzing certain nonlinear control problems.', ' It is based on quasi-linearization, which is the approximation of the non-linear system under investigation by a linear time-invariant (LTI) transfer function that depends on the amplitude of the input waveform.', ' By definition, a transfer function of a true LTI system cannot depend on the amplitude of the input function because an LTI system is linear.', ' Thus, this dependence on amplitude generates a family of linear systems that are combined in an attempt to capture salient features of the non-linear system behavior.', ' The describing function is one of the few widely applicable methods for designing nonlinear systems, and is very widely used as a standard mathematical tool for analyzing limit cycles in closed-loop controllers, such as industrial process controls, servomechanisms, and electronic oscillators.']], ['Nikolay Mitrofanovich Krylov', ['Nikolay Mitrofanovich Krylov (Russian: Никола́й Митрофа́нович Крыло́в , Ukrainian: Микола Митрофанович Крилов ) (29 November [O.S. 17 November] 1879 – May 11, 1955) was a Russian and Soviet mathematician known for works on interpolation, non-linear mechanics, and numerical methods for solving equations of mathematical physics.']], ['Master of Music', ['The Master of Music (M.M. or M.Mus.)', ' is, as an academic title, the first graduate degree in Music awarded by universities and conservatories.', ' The M.M. combines advanced studies in an applied area of specialization (usually performance in singing or instrument playing, composition, or conducting) with graduate-level academic study in subjects such as music history, music theory, or music pedagogy.', ' The degree, which takes one or two years of full-time study to complete, prepares students to be professional performers, conductors, and composers, according to their area of specialization.', ' The M.M. is often required as the minimum teaching credential for university, college, and conservatory instrumental or vocal teaching positions.']], ['New Chronology (Fomenko)', ['The New Chronology is a pseudohistorical theory which argues that the conventional chronology of Middle Eastern and European history is fundamentally flawed, and that events attributed to the civilizations of the Roman Empire, Ancient Greece and Ancient Egypt actually occurred during the Middle Ages, more than a thousand years later.', ' The central concepts of the New Chronology are derived from the ideas of Russian scholar Nikolai Morozov (1854–1946), although work by French scholar Jean Hardouin (1646–1729) can be viewed as an earlier predecessor.', ' However, the New Chronology is most commonly associated with Russian mathematician Anatoly Fomenko (born 1945), although published works on the subject are actually a collaboration between Fomenko and several other mathematicians.', ' The concept is most fully explained in \"History: Fiction or Science?\"', ', originally published in Russian.']], ['Doctor (title)', ['Doctor is an academic title that originates from the Latin word of the same spelling and meaning.', ' The word is originally an agentive noun of the Latin verb \"docēre \" ] \\'to teach\\'.', ' It has been used as an academic title in Europe since the 13th century, when the first doctorates were awarded at the University of Bologna and the University of Paris.', ' Having become established in European universities, this usage spread around the world.', ' Contracted \"Dr\" or \"Dr.\", it is used as a designation for a person who has obtained a Doctorate (e.g. PhD).', ' In many parts of the world it is also used by medical practitioners, regardless of whether or not they hold a doctoral-level degree.']], ['Georgian International Academy', ['Georgian International Academy (Georgian: საქართველოს საერთაშორისო აკადემია ) is a research and academic institution located in Tbilisi, Georgia.', ' The academy is one of the few Georgian institutions which awards the degree “Doctor Academician” – the highest academic title in Europe.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.776\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adf315c5542995ec70e8f8b', 'answer': 'Kylie Ann Minogue', 'question': 'Sudha Kheterpal who is  musician best know as the percussionist in Faithless has played with what singer who is recognized at the highest-selling Australian artist of all time by the Australian Recording industry Association?', 'supporting_facts': [['Sudha Kheterpal', 0], ['Kylie Minogue', 0], ['Kylie Minogue', 5]], 'context': [['Jessica Mauboy', ['Jessica Hilda Mauboy (born 4 August 1989) is an Australian R&B and pop singer, songwriter, and actress.', ' Born and raised in Darwin, Northern Territory, Mauboy rose to fame in 2006 on the fourth season of \"Australian Idol\"; she became the runner-up and subsequently signed a recording contract with Sony Music Australia.', ' After releasing a live album of her \"Idol\" performances and briefly being a member of the girl group Young Divas in 2007, Mauboy released her debut studio album, \"Been Waiting\", the following year.', ' It earned Mauboy her first number-one single \"Burn\", became the second highest-selling Australian album of 2009, and was certified double platinum by the Australian Recording Industry Association (ARIA).']], ['System of a Down discography', ['System of a Down is an American rock band formed by musicians of Armenian origin: vocalist Serj Tankian, guitarist Daron Malakian, bassist Shavo Odadjian, and drummer John Dolmayan in the mid-1990s.', ' They have released five studio albums, 16 singles, and 11 music videos. By the end of 1997, the group had signed to American Recordings, then distributed as Columbia Records.', ' The following year, they released their eponymous debut album, which peaked at #124 on the United States\\' \"Billboard\" 200 and #103 on the United Kingdom\\'s UK Albums Chart; it was certified platinum two years later by the Recording Industry Association of America\\xa0(RIAA), and gold by Canadian Recording Industry Association\\xa0(CRIA).', ' Their eponymous debut album produced a single for the song \"Sugar\", which reached the top 30 on the \"Billboard\" mainstream rock songs and alternative songs charts.', ' Their follow-up album, \"Toxicity\"\\xa0(2001), topped the US and Canadian charts, and also reached the top 10 in Australia, Finland, and New Zealand.', ' The album was certified triple platinum in its home country, and triple platinum in Australia by the Australian Recording Industry Association (ARIA), as well as double platinum by CRIA in Canada.', ' \"Toxicity\" produced singles for the title track, \"Chop Suey!', '\", and \"Aerials\".', ' The last of these peaked at number one on the \"Billboard\" Mainstream Rock Songs and Alternative Songs charts.']], ['Australian Recording Industry Association', ['The Australian Recording Industry Association (ARIA) is a trade group representing the Australian recording industry which was established in 1983 by six major record companies, EMI, Festival, CBS, RCA, WEA and Universal replacing the Association of Australian Record Manufacturers (AARM) which was formed in 1956.', ' It oversees the collection, administration and distribution of music licenses and royalties.']], ['Kylie Minogue', ['Kylie Ann Minogue, ( ; born 28 May 1968), often known simply as Kylie, is an Australian singer, songwriter, dancer and actress.', ' She achieved recognition starring in the Australian soap opera \"Neighbours\", where she played tomboy mechanic Charlene Robinson.', \" Appearing in the series for two years, Minogue's character married Scott Robinson (Jason Donovan) in an episode viewed by nearly 20 million people in the United Kingdom making it one of the most watched Australian TV episodes ever.\", ' Since then, Minogue has been a recording artist and has achieved commercial success and critical acclaim in the entertainment industry.', ' Minogue has been recognised with several honorific nicknames including \"Princess of Pop\" and \"Goddess of Pop\".', ' She is recognised as the highest-selling Australian artist of all time by the Australian Recording Industry Association (ARIA).']], ['ARIA Music Awards', ['The Australian Recording Industry Association Music Awards (commonly known informally as ARIA Music Awards or ARIA Awards) is an annual series of awards nights celebrating the Australian music industry, put on by the Australian Recording Industry Association (ARIA).', ' The event has been held annually since 1987 and encompasses the general genre-specific and popular awards (these are what is usually being referred to as \"the ARIA awards\") as well as Fine Arts Awards and Artisan Awards (held separately from 2004), Lifetime Achievement Awards and ARIA Hall of Fame – held separately from 2005 to 2010 but returned to the general ceremony in 2011.', ' For 2010, ARIA introduced public voted awards for the first time.']], ['List of number-one albums of 2009 (Australia)', ['The highest-selling albums in Australia are ranked in the Australian Recording Industry Association albums chart, also known as the ARIA Charts, published by the Australian Recording Industry Association (ARIA).', ' The data are compiled from a sample that includes music stores, music departments at electronics and department stores, and Internet sales (in other words, both digital as well as CD sales).', ' ARIA also issues a weekly singles chart.', ' In addition, data from these weekly charts are used to compile an end of year albums and singles chart.']], ['List of number-one albums of 2008 (Australia)', ['The highest-selling albums in Australia are ranked in the Australian Recording Industry Association albums chart, also known as the ARIA Charts, published by the Australian Recording Industry Association (ARIA).', ' The data are compiled from a sample that includes music stores, music departments at electronics and department stores and Internet sales, in other words, both digital as well as CD sales.', ' ARIA also issues a weekly singles chart and an end of year albums and singles chart, among other charts.']], ['Sudha Kheterpal', ['Sudha Kheterpal is a British-Indian musician best known as the percussionist in Faithless.', ' In 2008 she toured with The Return of the Spice Girls.', ' She has also played with K-Klass, Kylie Minogue, Melanie Williams, Jo Roberts, Corduroy, Mark Morrison, Rae and Christian, Ian Brown, Talvin Singh, and Dido.']], ['Stan Walker', ['Stan Walker (born 23 October 1990) is an Australian-New Zealand recording artist, actor, and television personality.', ' In 2009, Walker was the winner of the seventh and last season of \"Australian Idol\".', ' He subsequently signed a recording contract with Sony Music Australia.', ' In December 2009, Walker released his debut studio album, \"Introducing Stan Walker\", which included the hit single, \"Black Box\".', ' The album debuted at number three on the Australian ARIA Albums Chart and was certified platinum by the Australian Recording Industry Association (ARIA).', ' It also appeared on the New Zealand Albums Chart at number two and was certified triple platinum by the Recording Industry Association of New Zealand (RIANZ).']], ['Set It Off (Timomatic song)', ['\"Set It Off\" is the debut single by Australian recording artist Timomatic, released digitally on 18 November 2011, as the lead single from his self-titled second studio album.', ' It was written by Timomatic and DNA Songs, who also produced the track.', ' Timomatic stated that the song is \"about having fun on the dance floor\" and loving life.', ' \"Set It Off\" peaked at number two on the ARIA Singles Chart and was certified four times platinum by the Australian Recording Industry Association.', ' It also appeared on the New Zealand Singles Chart at number 14 and was certified gold by the Recording Industry Association of New Zealand.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.778\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8dde01554299653c1aa159', 'answer': 'USS \"Enterprise\"', 'question': 'In the \"Star Trek\" franchise, DeForest Kelley portrayed a character aboard which starship? ', 'supporting_facts': [['James T. Kirk', 2], ['James T. Kirk', 3], ['Leonard McCoy', 0], ['Leonard McCoy', 1]], 'context': [['List of Star Trek: The Original Series episodes', ['Created by Gene Roddenberry, the science fiction television series \"\" (which eventually acquired the retronym \"Star Trek: The Original Series\") starred William Shatner as Captain Kirk, Leonard Nimoy as Mr. Spock, and DeForest Kelley as Dr. Leonard \"Bones\" McCoy aboard the fictional Federation starship USS \"Enterprise\".', ' The series originally aired from September 1966 through June 1969 on NBC.']], ['Hikaru Sulu', ['Hikaru Kato Sulu is a fictional character in the \"Star Trek\" media franchise.', ' Originally known simply as \"Sulu\", he was portrayed by George Takei in the .', ' Sulu also appears in the , the first six \"Star Trek\" movies, one episode of \"\", and in numerous books, comics, and video games.', ' Sulu\\'s first name, \"Hikaru\", appeared in a 1981 novel well over a decade after the original series had ended.', ' John Cho assumed the role of the character in both the 2009 film \"Star Trek\" and its sequels, \"Star Trek Into Darkness\" and \"Star Trek Beyond\".']], ['Star Trek III: The Search for Spock', ['Star Trek III: The Search for Spock is a 1984 American science fiction film directed by Leonard Nimoy and based on the created by Gene Roddenberry.', ' It is the third film in the \"Star Trek\" film series, and is the second part of a three-film story arc that begins with \"\" (1982) and concludes with \"\" (1986).', ' After the death of Spock (Nimoy), the crew of the USS \"Enterprise\" returns to Earth.', ' When James T. Kirk (William Shatner) learns that Spock\\'s spirit, or katra, is held in the mind of Dr. Leonard \"Bones\" McCoy (DeForest Kelley), Kirk and company steal the \"Enterprise\" to return Spock\\'s body to his home planet.', ' The crew must also contend with hostile Klingons led by Kruge (Christopher Lloyd) who are bent on stealing the secrets of a powerful terraforming device.']], ['DeForest Kelley', ['Jackson DeForest Kelley (January 20, 1920 – June 11, 1999) was an American actor, screenwriter, poet and singer known for his roles in Westerns and as Dr. Leonard \"Bones\" McCoy of the USS\\xa0\"Enterprise\" in the television and film series \"Star Trek\".']], ['Leonard McCoy', ['Dr. Leonard H. \"Bones\" McCoy is a character in the American science fiction franchise \"Star Trek\".', ' First portrayed by DeForest Kelley in the , McCoy also appears in the , six \"Star Trek\" films, the pilot episode of \"\", and in numerous books, comics, and video games.', ' Karl Urban assumed the role of the character in the 2009 film \"Star Trek\", and its sequels, 2013\\'s \"Star Trek Into Darkness\" and 2016\\'s \"Star Trek Beyond\".']], ['Star Trek Spaceflight Chronology', ['Star Trek Spaceflight Chronology is a 1980 book written and edited by Stan and Fred Goldstein, and illustrated by Rick Sternbach.', ' At the time of its publication it was the official history of the \"Star Trek\" universe.', ' The first season of \"\" used references and dates that indicated that the \"Star Trek Spaceflight Chronology\" was no longer being followed, and it was eventually replaced by \"Star Trek Chronology\" as the official history of the \"Star Trek\" universe.', ' In 2006, Pocket Books published \"Voyages of Imagination\", which expanded \"Star Trek Chronology\" to include the events of all of the \"Star Trek\" novels.']], ['James T. Kirk', ['James Tiberius \"Jim\" Kirk is a fictional character in the \"Star Trek\" franchise.', ' Kirk first appears in \"\" and has been portrayed in numerous films, books, comics, webisodes, and video games.', ' As the captain of the starship USS \"Enterprise\", Kirk leads his crew as they explore \"new worlds, where no man has gone before\".', ' Often, the characters of Spock and Leonard McCoy act as his logical and emotional sounding boards, respectively.']], ['Star Trek', ['Star Trek is an American science fiction media franchise based on the television series created by Gene Roddenberry.', ' The first television series, simply called \"Star Trek\" and now referred to as \"\", debuted in 1966 and aired for three seasons on the television network NBC.', ' It followed the interstellar adventures of Captain James T Kirk (William Shatner) and his crew aboard the starship USS \"Enterprise\", a space exploration vessel, built by the interstellar federal republic United Federation of Planets in the twenty-third century.', ' The \"Star Trek\" canon of the franchise includes \"The Original Series\", an animated series, five spin-off television series, and its film franchise in addition to further adaptations made in several media since the original.']], ['Spock', ['Spock is a fictional character in the \"Star Trek\" media franchise.', ' Spock was first portrayed by Leonard Nimoy in the , and also appears in the , a two-part episode of \"\", eight of the \"Star Trek\" feature films, and numerous Star Trek novels, comics, and video games.', ' In addition, numerous actors portrayed the various stages of Spock\\'s rapid growth, due to the effects of the Genesis Planet, in the 1984 Star Trek film \"\".', ' In the 2009 film \"Star Trek\", Nimoy reprised his role with Zachary Quinto, who depicted a younger version of the character, existing within an alternate timeline.', ' Both reprised their roles in the 2013 sequel \"Star Trek Into Darkness\" and Quinto reprised his role again in 2016\\'s \"Star Trek Beyond\".']], ['Star Trek: Starship Creator', ['Star Trek: Starship Creator is a computer-based vehicle simulation game developed by Imergy and released by Simon & Schuster Interactive in 1998 for both Microsoft Windows and Mac OS, based on the official license of the \"Star Trek\" franchise.', ' Elements in the game were created in conjunction with the technical advisers for the series and films, such as Mike and Denise Okuda.', ' The gameplay in \"Starship Creator\" allowed for the player to equip and crew a series of different starship classes from across the \"Star Trek\" universe, including those from both the various series and the film series.', ' An expanded version was subsequently released as Star Trek: Starship Creator Deluxe in late 1999 which added further starships, missions and customization.', ' The official website also contained downloads for the characters from the \"\" series of books.', ' Reception by critics for \"Starship Creator\" was negative, with criticism directed at the gameplay and graphics, and the suggestion was made that the game would only appeal to \"Star Trek\" fans.', ' A sequel followed in 2000 entitled \"\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.778\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7a3bed5542996c55b2dd45', 'answer': 'Deftones', 'question': 'Which California band, whose debut album Adrenaline appeared in 1995, has been referred to as \"the Radiohead of metal\"?', 'supporting_facts': [['Adrenaline (album)', 0], ['Deftones', 0], ['Deftones', 4]], 'context': [['Ryan Ferguson (musician)', ['Ryan Ferguson (born \\u20091975 ) is an indie rock guitarist and singer-songwriter, formerly of Southern California band No Knife.', ' In 2004 he contributed the acoustic track \"Wait for Me There\" to the compilation album \"San Diego Is Burning\".', ' Ferguson\\'s solo debut \"Three, Four\" was released in July 2005, winning a San Diego Music Award for \"Best Pop Record\" that year.', ' Critic Chris Nixon cited it as one of the best releases of the year for San Diego musicians. \"', 'The Sims 2\" featured an interpretation of the album\\'s lead track \"Suddenly\".', ' He went on to tour with American rock band Switchfoot, and released a full-length album entitled \"Only Trying to Help\" in 2007.']], ['Cream City', ['Cream City is debut album by funk/soul Los Angeles, California band Aalon.', ' Released in 1977 and produced by Jerry Goldstein, the album reached number 45 on the R&B albums chart in the US.', \" The group disbanded shortly after the album's release.\", ' It was reissued on CD by Thump Records in the mid-1990s, and the album has had a cult following ever since.']], ['Adrenaline (album)', ['Adrenaline is the debut studio album by American alternative metal band Deftones, released in 1995 by Maverick Records.', ' A hidden track on the album, \"Fist\", was produced by Ross Robinson, while the rest of the album was produced by Terry Date.']], ['F-Minus (band)', ['F-Minus was a hardcore punk band formed in 1995 in Huntington Beach, California, started by Jen Johnson and Brad Logan.', ' F-Minus was known for their dueling male and female vocals in songs that were sometimes as short as 12 seconds (\"Fuck You O.C.\").', ' Before breaking up in 2004, their last album was recorded by Steve Albini.', ' Throughout their career, they covered such bands as Antidote, Black Randy and the Metro Squad, 7 Seconds, Negative Approach, and Agnostic Front.', ' Brad Logan currently runs his own record label Blacknoise, and is also member of the New York band Leftöver Crack.', ' Jen Johnson currently is the designer and owner of clothing label E.C. Star, and also is a member of the California band Ammunition Affair.']], ['Given to the Rising', ['Given to the Rising is the ninth studio album from the Oakland, California band Neurosis, released on June 5, 2007.', ' The album is available in a standard jewel case, a limited-edition digipak, and a limited-edition double LP, all with the same track list. \"', 'Decibel Magazine\" listed \"Given to the Rising\" as the 76th best metal album of the decade.', ' A DVD documentary is also available from Neurot Records.', ' The artwork for the album, designed by Josh Graham, is a mixture of photos from Heroes Square in Budapest and drawings inspired by the place.', ' Coincidentally, singer/guitarist Steve Von Till and Josh Graham had separately thought of the idea of using Heroes Square as the artwork for the album.']], ['Noah Georgeson', ['Noah Georgeson is a Grammy winning musician, producer, and solo recording artist.', ' Georgeson\\'s debut album \"Find Shelter\" was released through Plain Recordings on November 28, 2006.', ' Born in San Anselmo, California, he moved with his family to Nevada City, California at the age of three.', ' Georgeson studied classical guitar and music composition, receiving his BA in composition from San Francisco State University in 2001, and, with a recommendation from Terry Riley, he attended Mills College, receiving his Master of Fine Arts in 2003.', ' While at Mills, Georgeson studied with Fred Frith, Pauline Oliveros, and Alvin Curran.', ' Georgeson first found popular success as a part of San Francisco band The Pleased, along with fellow member Joanna Newsom, whose debut album \"The Milk-Eyed Mender\" he produced.', ' As a musician, producer, and mixer, Georgeson has since worked with Devendra Banhart, The Strokes, Little Joy, Bert Jansch, Charlotte Gainsbourg, Robin Pecknold, Mason Jennings, Cedric Bixler, Adam Green, Os Mutantes, Adan Jodorowsky, Harper Simon, Flo Morrissey, Cate Le Bon, and Rodrigo Amarante.']], ['The Red &amp; the Black', ['The Red & the Black is a rock album by the Poway, California band Agent 51, released by Surfdog Records in 2003.', ' It was the band\\'s third album and was named \"best punk album\" at the 2003 San Diego Music Awards.', ' With it the band distanced themselves from their previous punk rock sound by incorporating much more of an influence of classic rock and heavy metal bands such as AC/DC, Motörhead and Def Leppard.', ' They also distanced themselves from the secret agent mythos they had created for themselves by ceasing to wear matching black suits in concert and by ceasing to write songs dealing with the existence of extraterrestrials and UFOs.', ' The song \"Loaded\" was originally titled \"Fuckin\\' Loaded\" but was changed for printing on the album sleeve.', ' They received airplay on local rock radio station 91X for the song \"She\\'s My Heroine.\"', \" The album was the band's last before their extended hiatus, and they would not reconvene until a reunion show in 2005.\", ' Agent 51 have not released any more albums since \"The Red & the Black\".']], [\"Just Keep Runnin'\", [\"Just Keep Runnin<nowiki>'</nowiki> is a punk rock album by the Poway, California band Agent 51.\", \" It was first released in 2000 on the band's own Suburban Hooligans Records label, then was re-released in 2001 by Adeline Records with additional tracks.\", \" It was the band's second album and expanded their punk rock sound to include influences of classic rock and heavy metal.\", \" It also greatly expanded the band's secret agent mythos, with the liner notes extensively detailing the comic book-style secret origins of the band and its members.\", ' According to the story, the band members were \"rogue secret agents with psychic, cybernetic and alien-enhanced abilities\" who were struggling against a secret government organization known as \"The Agency.\"', \" The Agency sought to use satellites to control the minds of Earth's population, and Agent 51's mission was to expose the Agency's secrets to the general public disguised as an ordinary rock band.\", ' The songs \"C.I.A.F.B.I.\" and \"Psychic Spies\" dealt directly with this theme.', ' The songs \"The Last Pirate Standing\", \"Free-Wheel\" and \"Who\\'s Gunna Riot?\"', ' had been part of the band\\'s live set for several years under the titles \"The Pirate Song\", \"Free-Wheel Burning\" and \"Riot.\"']], ['Against All Will', ['Against All Will is a rock quartet from Los Angeles, California, whose debut album lineup was formed in Spring 2009 by Jimmy Allen (formerly of Puddle of Mudd), Jeff Current (formerly of Seven Story Drop), Steve \"Boomstick\" Wilson (formerly of Dead Kennedys and t.A.T.u.), and Cello Dias (formerly of Soulfly).', ' Songs \"All About You\" and \"The Drug I Need\" from their debut album entitled A Rhyme & Reason ranked in the national rock radio top 50 in 2010.']], ['Deftones', ['Deftones is an American alternative metal band from Sacramento, California, U.S. Formed in 1988, the band was founded by Chino Moreno (lead vocals, rhythm guitar), Stephen Carpenter (lead guitar), Abe Cunningham (drums) and Dominic Garcia (bass).', \" During the group's first five years, the band's lineup changed several times, but stabilized in 1993 when Cunningham rejoined the group after his departure in 1990; by this time, Chi Cheng was the band's bassist.\", ' The lineup remained stable for fifteen years, with the exception of keyboardist and turntablist Frank Delgado being added in 1999.', ' The band is known as one of the most experimental groups to have come from the alternative metal music scene.', ' They are sometimes dubbed \"the Radiohead of metal\" by critics.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.778\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a83d7d05542992ef85e237a', 'answer': 'Liberty Enlightening the World', 'question': 'What is the name of the statue whose replica have been created in many landmarks world-wide such as near Pont de Grenelle in Paris?', 'supporting_facts': [['Pont de Grenelle', 0], ['Pont de Grenelle', 4], ['Replicas of the Statue of Liberty', 0]], 'context': [['Pont De Rennes bridge', [\"The Pont De Rennes bridge is located in the Brown's Race Historic District of Rochester New York at the base of the High Falls where it spans the Genesee River.\", ' The Pont De Rennes bridge formerly carried Platt Street over the river but was converted to pedestrian use in 1982 as part of a redevelopment of the High Falls area as an entertainment area.', \" The bridge was renamed the Pont De Rennes for Rochester's sister city Rennes in France as part of the conversion.\", ' The Pont De Rennes bridge provides unobstructed views of the High Falls and downstream gorge.']], ['Replicas of the Statue of Liberty', ['Hundreds of replicas of the Statue of Liberty (\"Liberty Enlightening the World\") have been created worldwide.']], ['Éleuthère Irénée du Pont', ['Éleuthère Irénée du Pont de Nemours (24 June 1771 – 31 October 1834), known as Irénée du Pont, or E. I. du Pont, was a French-American chemist and industrialist who founded the gunpowder manufacturer E. I. du Pont de Nemours and Company.', \" His descendants, the Du Pont family, have been one of America's richest and most prominent families since the 19th century, with generations of influential businessmen, politicians and philanthropists.\"]], ['Pont de Grenelle', ['The pont de Grenelle is a bridge in Paris, France, that crosses the Seine river.', \" It connects the city's 15th and 16th arrondissements, and passes through the Île aux Cygnes.\", ' Constructed of steel, it is a girder bridge.', ' The current bridge was constructed in 1966, replacing an earlier bridge that had stood since 1873.', ' The bridge passes behind a replica of the Statue of Liberty.']], ['Dupleix (Paris Métro)', ['Dupleix is an elevated station of the Paris Métro serving line 6 along \"Boulevard de Grenelle\" in the 15th arrondissement.', ' The track and station form an elevated viaduct in the centre of and above \"Boulevard de Grenelle\".', ' There is an open street market under the station twice a week.']], ['Magic-City', [\"Magic-City was an amusement park near Pont de l'Alma, two blocks east of the Eiffel Tower, in Paris, France from 1900 to 1934.\"]], ['Pont de Brotonne', ['The Brotonne Bridge (\"pont de Brotonne\") is a bridge in the region of Upper Normandy in France, situated between the cities of Le Havre and Rouen.', ' It has crossed the Seine since 1977, to the east of the commune of Caudebec-en-Caux.', ' Its construction was financed by the General council of Seine-Maritime for the purpose of opening up the Pays de Caux and assuring a connection between the commune of Yvetot and the A13 autoroute by way of the forêt de Brotonne (\"Brotonne forest\"), from which the bridge gets its name.', ' Only two bridges are located further downstream the Seine from the pont de Brotonne: the Pont de Tancarville and the Pont de Normandie.']], ['Musée de Radio France', ['The Musée de Radio France was a museum operated by Radio France and located in the Maison de Radio-France, near the Pont de Grenelle in the XVIe arrondissement at 116, avenue du Président Kennedy, Paris, France.', ' The museum was established in 1966, and contained a remarkable collection of radios and televisions from their origins to the present day, including the 1793 telegraph by Claude Chappe and early crystal radios.', \" The museum's 2000 objects include prototypes and commercial devices, archival documents, photographs, and manuscripts, replicas of early radio laboratories and studios, and exhibits featuring research by Edouard Branly, Lee de Forest, Heinrich Hertz, Guglielmo Marconi, James Clerk Maxwell, and Alexander Stepanovich Popov.\", ' In 2007, the museum was closed to the public due to the renovation of the Maison de Radio France.']], ['La Motte-Picquet – Grenelle', ['La Motte-Picquet – Grenelle is a station of the Paris Métro, at the interconnection of lines 6, 8 and 10 in the 15th \"arrondissement\", near the 7th \"arrondissement\".', ' The station combines underground and elevated platforms.', ' It is named after the \"Avenue de la Motte-Picquet\" and the \"Boulevard de Grenelle\", as the station is located at the intersection of these two streets.', ' It is a major Paris Metro interconnection on the Rive Gauche, and the most important west of Montparnasse.']], ['Pont de la Concorde (Paris)', [\"The Pont de la Concorde is an arch bridge across the River Seine in Paris connecting the Quai des Tuileries at the Place de la Concorde (on the Right Bank) and the Quai d'Orsay (on the Left Bank).\", ' It has formerly been known as the Pont Louis XVI, Pont de la Révolution, Pont de la Concorde, Pont Louis XVI again during the Bourbon Restoration (1814), and again in 1830, Pont de la Concorde, the name it has retained to this day.', ' It is served by the Metro stations Assemblée nationale and Concorde.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.780\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae1c214554299234fd042ed', 'answer': 'Nicholas John \"Nic\" Cester', 'question': 'Which  Australian singer-songwriter wrote  Cold Hard Bitch', 'supporting_facts': [['Cold Hard Bitch', 1], ['Nic Cester', 0]], 'context': [['Cold Hard Bitch', ['\"Cold Hard Bitch\" is the fourth single (second in the United States) by the Australian rock group, Jet, from their 2003 album, \"Get Born\".', ' It was released in March 2004 and was written by band-members Chris Cester, Nic Cester, and Cameron Muncey.', \" On the ARIA Singles Chart in the group's native country, it reached the top 40.\"]], ['Rollover DJ', ['\"Rollover DJ\" is the second single (except in the United States, where it was the third, after \"Cold Hard Bitch\") by the Australian rock band Jet, from their debut album \"Get Born\" (14 September 2003).', ' It was released in November, two months after the album, and was promoted with two different music videos.', ' It reached the Top\\xa040 on both the ARIA Singles Chart and the UK Singles Chart.']], ['Cameron Muncey', ['Cameron Thane Muncey (born 8 February 1980) is an Australian guitarist and vocalist.', ' He is the mainstay lead guitarist and one of the songwriters of Melbourne-based rock band Jet which formed in 2001.', ' Muncey co-wrote many of Jet\\'s hits with Nic and Chris Cester, including \"Are You Gonna Be My Girl\", \"Radio Song\", \"Put Your Money Where Your Mouth Is\" and \"Cold Hard Bitch\".']], ['Cold Hard Truth', ['Cold Hard Truth is the 56th studio album by American country music singer George Jones.', ' The album was released on June 22, 1999 on the Asylum label.']], ['Nic Cester', ['Nicholas John \"Nic\" Cester (born 6 July 1979) is an Australian singer-songwriter and guitarist, known for being the frontman in rock band Jet alongside his younger brother Chris.', ' Cester is also a founder of the Australian supergroup The Wrights.', ' Jet\\'s track \"Are You Gonna Be My Girl\", has won APRA Awards for \\'Most Performed Australian Work Overseas\\' in 2006 and 2007.']], ['Cold Hard Want', ['Cold Hard Want is the fifth full-length album by alternative rock band House of Heroes.', ' It was released on Gotee Records on July 10, 2012.', ' House of Heroes entered Smoakstack Studios on December 12, 2011 in order to record \"Cold Hard Want\".', ' The band went with producer Paul Moak, who has produced artists such as Seabird, Lovedrug, Mat Kearney, after going with producer Mark Lee Townsend for the previous two albums.', ' As of February 22, 2012, the record has been completely recorded and mastered.']], ['Dirty Sweet', ['Dirty Sweet is the debut four-track extended play released by Australian rock band Jet in November 2002 on Rubber Records.', ' It was re-recorded and re-issued on 6 May 2003 by Elektra Records.', ' All tracks on the EP are also on the band\\'s debut album, \"Get Born\", which followed on 14 September.', ' Two tracks, originally on the EP, were later issued as singles from \"Get Born\", \"Rollover DJ\" (November) and \"Cold Hard Bitch\" (March 2004).']], ['Ameibo', ['Ameibo is a web-based video on demand (VOD) company that allows its users to legally download and share movies with other paying customers.', ' It is the only website in the world that pays you cold hard cash when you legally share/seed the movies you Buy or Rent from the website.', ' The company aims to combat online movie piracy by imitating the common BitTorrent (protocol) technique as a way for users to only share the content with other paying users.']], ['Choices (Billy Yates song)', ['\"Choices\" is a Grammy-award winning country music song written by Billy Yates and Mike Curtis, first recorded by Yates on his 1997 self-titled album for Almo Sounds.', ' It was later covered by George Jones, who released as the first single from his album \"The Cold Hard Truth\" on May 8, 1999, and it peaked at number 30 on the \"Billboard\" country charts.']], ['House of Heroes', ['House of Heroes is an alternative rock band from Columbus, Ohio.', ' They have released six albums: \"What You Want Is Now\" (2003), \"House of Heroes\" (2005), \"The End Is Not the End\" (2008), \"Suburba\" (2010), \"Cold Hard Want\" (2012), and \"Colors\" (2016).', ' The band also released the album \"Ten Months\" (2001) under their original name, No Tagbacks, which had more of a punk sound than their releases as House of Heroes.', ' They also re-released their self-titled record under the name of \"Say No More\" (2006).', ' The band is composed of Tim Skipper, Colin Rigsby, A.J. Babcock, and Eric Newcomer.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.780\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae4eb9255429908b63264ba', 'answer': 'Mohawk, Onondaga, Oneida, Cayuga, Seneca, and Tuscarora', 'question': 'What were the nations after 1722 that composed a culture that was extensively studied by William N. Fenton?', 'supporting_facts': [['William N. Fenton', 0], ['Iroquois', 1]], 'context': [['Bimal N. Patel', ['Bimal N. Patel is a Professor of Public International Law and the current Director of the Gujarat National Law University, Gandhinagar.', ' He was appointed by a High Level Committee headed by the then Chief Justice of India, K G Balakrishnan, at the Supreme Court of India premises.', \" The Government of India has also recently appointed him as a member of the 21st Law Commission of India along with Justice Balbir Singh Chauhan, retired Judge of the Hon'ble Supreme Court of India as its chairperson.\", ' Prof. Patel is a former International civil servant, scholar and academician of international law and diplomacy.An acclaimed international law jurist, he has extensively studied, researched, commented and published works on the administrative, procedural and substantive jurisprudence of the International Court of Justice (ICJ), International Tribunal for the Law of the Sea (ITLOS), International Criminal Tribunal for former Yugoslavia (ICTY) and International Labour Organisation Administrative Tribunal (Geneva).', ' His publications on India and International Law and Responsibility of International Organisations are reviewed and referred by international law scholars and journals across the world.', ' He has published, edited several books, research papers/articles/surveys in leading academic and international law journals.', ' He has been involved in drafting several national and state primary and secondary legislations, regulations, rules and holds the distinction as one of the first Indians to serve at the International Labour Organization Administrative Tribunal (Geneva).', ' He has delivered numerous lectures, including one at Cambridge University, UK, and has received several honours.', ' He has served at the Organisation for the Prohibition of Chemical Weapons,Hague, Netherlands.']], ['Ashanti Empire', ['The Ashanti (also spelled Asante) Empire (1701–1957) was an Akan empire and kingdom in what is now modern-day Ghana.', ' The Ashanti Empire expanded from Ashanti to include the Brong-Ahafo, Central region, Eastern region, Greater Accra region, and Western region, of present-day Ghana.', ' The Ashanti benefited from early firearm adoption.', ' Combined with effective strategy, they fashioned an empire that stretched from central Ghana to the present-day Ivory Coast.', \" Due to the empire's military prowess, wealth, architecture, sophisticated hierarchy and culture, Ashanti has been extensively studied and has more historiographies by European, primarily British, authors than almost any other indigenous culture of Sub-Saharan Africa.\"]], ['William N. Fenton', ['William N. Fenton (December 15, 1908 – June 17, 2005) was an American scholar and writer known for his extensive studies of Iroquois history and culture.', ' He started his studies of the Iroquois in the 1930s and published a number of significant works over the following decades.', ' His final work was published in 2002.', ' During his career, Fenton was director of the New York State Museum and a professor of anthropology at the State University of New York.']], ['William N. Rhodes', ['William N. Rhodes was an American airforce Technical Sergeant in World War II.', ' On March 31, 1945, TSgt.', \" William N. Rhodes' aircraft was engaged in a mission to take out a primary target (oil refinery) at Ziet, Germany.\", ' During that engagement his B-17 aircraft was hit by enemy fire.', ' The number three engine oil supply line was cut by flak, and the landing gear was hit and jammed.', ' Flak also damaged an engine housing causing that prop to be shut down and feathered.', ' The Aircraft was able to maintain an altitude of 17,500 feet and began its journey back to England when two jet propelled German fighters attacked.', ' These two German aircraft were sighted and immediately reported to the pilot by TSgt.', ' Rhodes.', \" The B-17 was hit during the German fighter attack inflicting extensive damage to the aircraft's number three fuel tank, causing it to explode and tossed the right wing violently.\", ' Following this hit the aircraft went into a tight downward spin, within just a few thousand feet the tail section of the aircraft blew off causing the aircraft to level off slightly and continue falling in a shallow spin.', ' This presented the opportunity for the Navigator, Turret Gunner, Co-Pilot and TSgt.', ' Rhodes to bail out.', ' TSgt.', ' Rhodes and three other crew members landed near Biberach, Germany.', ' Of the nine original crew members on the B-17, only four survived the aerial encounter.', ' Upon landing, TSgt.', ' Rhodes and the other survivor’s were captured by German troops waiting on the ground, searched and taken to a Luftwaffe camp where they were processed as Prisoners of war.']], ['Iroquois', ['The Iroquois ( or ) or Haudenosaunee ( ) are a historically powerful northeast Native American confederacy.', ' They were known during the colonial years to the French as the \"Iroquois League,\" and later as the \"Iroquois Confederacy,\" and to the English as the \"Five Nations\" (before 1722), and later as the \"Six Nations,\" comprising the Mohawk, Onondaga, Oneida, Cayuga, Seneca, and Tuscarora peoples.']], ['Near polygon', ['In mathematics, a near polygon is an incidence geometry introduced by Ernest E. Shult and Arthur Yanushka in 1980.', ' Shult and Yanushka showed the connection between the so-called tetrahedrally closed line-systems in Euclidean spaces and a class of point-line geometries which they called near polygons.', ' These structures generalise the notion of generalized polygon as every generalized 2\"n\"-gon is a near 2\"n\"-gon of a particular kind.', ' Near polygons were extensively studied and connection between them and dual polar spaces was shown in 1980s and early 1990s.', ' Some sporadic simple groups, for example the Hall-Janko group and the Mathieu groups, act as automorphism groups of near polygons.']], ['Candicine', ['Candicine is a naturally occurring organic compound that is a quaternary ammonium salt with a phenethylamine skeleton.', ' It is the N,N,N-trimethyl derivative of the well-known biogenic amine tyramine, and, being a natural product with a positively charged nitrogen atom in its molecular structure, it is classed as an alkaloid.', ' Although it is found in a variety of plants, including barley, its properties have not been extensively studied with modern techniques.', ' Candicine is toxic after parenteral administration, producing symptoms of neuromuscular blockade; further details are given in the \"Pharmacology\" section below.']], ['Kunda culture', ['Kunda Culture, originating from the Swiderian culture, comprised mesolithic hunter-gatherer communities of the Baltic forest zone extending eastwards through Latvia into northern Russia, dating to the period 8500–5000 BC according calibrated radiocarbon dating.', ' It is named after the Estonian town of Kunda, about 110 km east of Tallinn along the Gulf of Finland, near where the first extensively studied settlement was discovered on Lammasmäe Hill and in the surrounding peat bog.', ' The oldest known Kunda culture settlement in Estonia is Pulli.', ' The Kunda Culture was succeeded by the Narva culture, who used pottery and showed some traces of food production.']], ['Oliver Phase', ['The Oliver Phase is the name for a Late Woodland Native American culture that flourished from 1200 and 1450 CE along the east and west forks of the White River in central and southern Indiana.', ' The Oliver Phase is of the Western Basin Tradition which includes the Springwells Phase, the Younge Phase, and the Riviere au Vase Phase.', ' Oliver people were village dwelling farmers with a heavy reliance on maize, very similar to other Late Woodland peoples in the area the Oneota, Fort Ancient, and Monongahela cultures.', ' The name was originally coined by archaeologist James B. Griffin in 1946 to describe a Late Woodland ceramic complex centered in Hamilton and Marion counties in the valley of the West Fork of the White River first extensively studied at the Bowen site.']], ['Nematostella', ['Nematostella is a genus of sea anemones in the family Edwardsiidae.', ' There are three species.', ' The best known is the starlet sea anemone (\"N. vectensis\"), which has been extensively studied as a model organism in fields such as genetics, evolution, and ecology.', ' The defining morphological apomorphy of \"Nematostella\" is the presence of nematosomes.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.781\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab5c8c5554299637185c60d', 'answer': 'no', 'question': 'Are Harry Everett Smith and Vladimir Danilevich both from Russia?', 'supporting_facts': [['Harry Everett Smith', 0], ['Vladimir Danilevich', 0]], 'context': [['Early Abstractions', ['Early Abstractions is a collection of seven short animated films created by Harry Everett Smith between 1939 and 1956.', ' Each film is between two and six minutes long, and is named according to the chronological order in which it was made.', ' The collection includes \"Numbers 1–5\", \"7\", and \"10\", while the missing \"Numbers 8\" and \"9\" are presumed to have been lost.']], ['Bud Abell', ['Harry Everett \"Bud\" Abell (born December 21, 1940) is a former American football linebacker in the American Football League for the Kansas City Chiefs.', ' He played college football at the University of Missouri.']], ['Vladimir Danilevich', ['Vladimir Petrovich Danilevich (Russian: Владимир Петрович Данилевич ; 4 September 1924 — 9 October 2001) was well-known Soviet and Russian Animator: who successfully worked as the film director, the screenwriter, the art director and the animator.']], ['Harry Everett Townsend', ['Harry Everett Townsend (1879–1941) was a war artist for the United States Army during World War I.']], ['Harry Everett Smith', ['Harry Everett Smith (May 29, 1923 in Portland, Oregon – November 27, 1991 in New York City) was a visual artist, experimental filmmaker, record collector, bohemian, mystic, and largely self-taught student of anthropology.', ' Smith was an important figure in the Beat Generation scene in New York City, and his activities, such as his use of mind-altering substances and interest in esoteric spirituality, anticipated aspects of the Hippie movement.', ' Besides his films, Smith is widely known for his influential \"Anthology of American Folk Music\", drawn from his extensive collection of out-of-print commercial 78 rpm recordings.']], ['Vaniusha and The Space Pirate', ['Vaniusha and The Space Pirate (Russian: Ванюша и космический пират ) is a 1991 Soviet Russian stop-motion animation film by Vladimir Danilevich.', ' This film was produced by Soyuzmultfilm studio.', ' <br>The film is about The Friendly Newcomer from another planet.', '<br> The film is The Third Film of the tetralogy, which tells about the adventures of The Newcomer Vaniusha and his friends.', ' Other three films called \"The Newcomer in The Cabbage\", \"Vaniusha The Newcomer\" and \"Vaniusha and The Giant\".']], ['Vaniusha and The Giant', ['Vaniusha and The Giant (Russian: Ванюша и великан ) is a 1993 Russian stop-motion animation film by Vladimir Danilevich.', ' This film was produced by Soyuzmultfilm studio.', ' <br>The film is about The Friendly Newcomer from another planet.', '<br> The film is the fourth film of the tetralogy, which tells about the adventures of The Newcomer Vaniusha and his friends.', ' The first three films are \"The Newcomer in The Cabbage\", \"Vaniusha The Newcomer\" and \"Vaniusha and The Space Pirate\".']], ['The Newcomer in The Cabbage', ['The Newcomer in The Cabbage (Russian: Пришелец в капусте ) is a 1989 short Soviet Russian stop-motion animation film by Vladimir Danilevich.', ' It is the first film of a tetralogy about the adventures of The Newcomer Vaniusha and his friends.', ' It was followed by \"Vaniusha The Newcomer\", \"Vaniusha and The Space Pirate\" and \"Vaniusha and The Giant\".']], ['Vaniusha The Newcomer', ['Vaniusha The Newcomer (Russian: Пришелец Ванюша ) is a 1990 Soviet Russian stop-motion animation film by Vladimir Danilevich and Olga Panokina.', ' It was produced by Soyuzmultfilm studio.', ' <br>The film is about The Friendly Newcomer from another planet.', '<br> It is the second film of the tetralogy, which tells about the adventures of The Newcomer Vaniusha and his friends.', ' The other three films are \"The Newcomer in The Cabbage\", \"Vaniusha and The Space Pirate\" and \"Vaniusha and The Giant\".']], ['Heaven and Earth Magic', ['Heaven and Earth Magic (also called \"Number 12\", \"The Magic Feature\", or \"Heaven and Earth Magic Feature\") is an American avant garde feature film made by Harry Everett Smith.', ' Originally released in 1957, it was re-edited several times and the final version was released in 1962.', ' The film primarily uses cut-out-animated photographs.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.782\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a87b6215542996e4f3088d6', 'answer': 'no', 'question': 'Are Yoo-hoo and Faygo both carbonated drinks?', 'supporting_facts': [['Yoo-hoo', 0], ['Faygo', 0]], 'context': [['Ceylon Cold Stores', ['Ceylon Cold Stores (CCS), trading as Elephant House, is a Sri Lankan company which produces carbonated drinks, ice cream and processed meat products.', ' Despite competition from global competitors such as Coca-Cola and Pepsi, Elephant Soft Drinks remains the market leader in Sri Lanka.']], ['Schorle', ['Schorle is a German beverage made by diluting juice or wine with carbonated water or lemonade (lemon-lime soda).', ' The most common variety is Apfelschorle (made from apple juice and sparkling mineral water).', ' Large bottles of Schorle can be found at most grocers, stores, supermarkets and anywhere else where carbonated drinks are sold, next to the soda.', ' Due to its dilution it is less sweet or alcoholic than the original beverage, making it better suited as a refreshment on hot summer days or as an alternative to beer at the biergarten or .']], ['Faygo', ['Faygo Beverages, Inc., is a soft drink company headquartered in Detroit, Michigan.', ' The beverages produced by the company, branded as Faygo or Faygo Pop, are distributed in the Midwest, Mid-Atlantic, and Central Southern regions of the United States, as well as southern Canada.', ' Faygo is imported in Europe by American Fizz, an official distributor of Faygo.', ' Faygo Beverages, Inc., is a wholly owned subsidiary of the National Beverage Corporation, started in Detroit, Michigan, in 1907 as Feigenson Brothers Bottling Works.']], ['Grape soda', ['Grape soft drinks (also known as grape drink, grape soda or grape pop in certain regions of the US) are typically sweetened carbonated drinks with a grape flavor.']], ['Carbonated drink', ['Carbonated drinks are beverages that contain dissolved carbon dioxide.', ' The dissolution of CO in a liquid, gives rise to \"fizz\" or \"effervescence\".', ' The process usually involves carbon dioxide under high pressure.', ' When the pressure is removed, the carbon dioxide is released from the solution as small bubbles, which causes the solution to become effervescent, or fizzy.', ' A common example is the dissolving of carbon dioxide in water, resulting in carbonated water.', ' Carbon dioxide is only weakly soluble in water, therefore it separates into a gas when the pressure is released.']], ['Tampico Beverages', ['Tampico Beverages is a manufacturer of juice concentrates, carbonated drinks, iced tea and gelatin.', ' Selling under the TAMPICO brand name, it is available in the United States and more than 55 countries around the globe.', ' Tampico Beverages is wholly owned by Houchens Industries, Inc, since 2008.']], ['Banta', ['Banta also known as Fotash Jawl in Bengali, Goli Soda (\"Goli\" = spherical object in Hindi) or Goti Soda (\"Goti\" = marble in Hindi) is a colloquial term for a carbonated lemon or orange-flavoured soft drink popular in India.', ' Though the origin of its name is from Punjabi word for marble (banta), Banta has been sold since the late 19th century, long before popular carbonated drinks arrived.', ' The drink is often sold mixed with lemon juice, crushed ice, chaat masala and kala namak (black salt) as a carbonated variant of popular lemonades \"shikanjvi\" or \"jal-jeera\".', ' It is available at street-sellers known as \"bantawallahs\" at prices ranging from - .']], ['Codd-neck bottle', ['A Codd bottle is a type of bottle used for carbonated drinks.', ' It has a unique closing design based on a glass marble which is held against a rubber seal, which sits within a recess in the lip.']], ['SodaStream', ['SodaStream International Ltd. (NASDAQ:\\xa0SODA ) is an Israeli drinks company best known as the maker of the consumer home carbonation product of the same name.', ' The device, like a soda syphon, carbonates water by adding carbon dioxide from a pressurized cylinder to create soda water (or carbonated water) to drink.', ' The company also sells more than 100 types of concentrated syrups and flavourings to make carbonated drinks.']], ['Yoo-hoo', ['Yoo-hoo is an American brand of chocolate beverage that originated in New Jersey in 1926 and that is currently manufactured by Dr. Pepper Snapple Group.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.782\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7f8b8c5542995d8a8ddea1', 'answer': 'a pair of dice', 'question': 'The term snake eyes is used in a particular casino game that uses what game pieces?', 'supporting_facts': [['Snake eyes', 0], ['Craps', 0]], 'context': [['Game piece (music)', ['Game piece is a concept of experimental music having its roots with composers Iannis Xenakis, Christian Wolff , John Zorn and Mathius Shadow-Sky.', ' Game pieces may be considered controlled improvisation.', ' An essential characteristic is that there is no pre-arranged sequence of events.', ' They unfold freely according to certain rules, like in a sports game.', ' Therefore, game pieces have elements of improvisation.', ' A number of methods can be used to determine the direction and evolution of the music, including hand gestures and shuffled cards, as in his file-card compositions.', ' Zorn\\'s game piece \"Cobra\", which has been recorded several times for various labels, uses a combination of cards and gestures and can be performed by an ensemble of any size and composition.', ' Zorn\\'s game pieces, written in the late 1970s and mid-1980s, include \"Cobra\", \"Hockey\", \"Lacrosse\", and \"Xu Feng\".', ' His file-card compositions include \"Spillane\" and \"Godard\".', '.', \" Mathius Shadow-Sky (born 1961) developed music gaming system founded on Roger Caillois, Gilles Deleuze, and Lewis Caroll's concepts to create new 'scoring' for music.\", \" Starting in 1980 with Ludus Musicae Temporarium for an 'archisonic lamps consort' , followed by several music games among them: The Ephemerodes Card of Chrones in 1984 for a broken piano orchestra, a temporal music game based on elastic rhythms interactions (within nonoctave scales for sliding morphing harmony) .\"]], ['Xu Feng (album)', [\"Xu Feng: John Zorn's Game Pieces Volume 1 is an album by American composer and saxophonist/multi-instrumentalist John Zorn consisting of game pieces.\", ' It features improvisations performed by an ensemble of pairs of musicians using the same instruments: Chris Brown and David Slusser on electronics; Fred Frith and John Schott on guitars; and Dave Lombardo (from Slayer) and William Winant on drums and percussion.', ' The album is titled after Xu Feng, a Taiwanese actress featured in many martial arts films who appears on the cover artwork.']], ['Pichenotte', ['Pichenotte is a French Canadian tabletop game, with a board, game pieces and rules similar to carrom.', ' Used more broadly, the term is a general name for tabletop games played with small (usually wooden) pieces that are flicked using the thumb and index (or middle) finger, including such games as carrom, sharing a similarity in that their mechanics lie somewhere between pocket billiards and shuffleboard.', ' The term is sometimes also mistakenly used as the actual name of other games of this class, such as carrom and crokinole.', ' Commercially produced boards are available, some under the trade name Pinnochi.', ' The game is sometimes referred to as \"piche\" or \"pish\".']], ['Snake Eyes (G.I. Joe)', ['Snake Eyes (also released as \"Snake-Eyes\") is a fictional character from the \"\" toyline, comic books, and cartoon series.', ' He is one of the original and most popular members of the G.I. Joe Team, and is most known for his relationships with Scarlett and Storm Shadow.', ' Snake Eyes is one of the most prominent characters in the \"G.I. Joe: A Real American Hero\" franchise, having appeared in every series of the franchise since its inception.', ' He is portrayed by Ray Park in the 2009 live-action film \"\", and the 2013 sequel \"\".']], ['Storm Shadow (G.I. Joe)', ['Storm Shadow is a fictional character from the toyline, comic books and cartoon series.', \" He is best known as the Cobra Commander's ninja bodyguard, and for his history with fellow ninja Snake Eyes.\", ' Throughout their history, he has changed sides several times; conflicted in loyalties between Cobra, G.I. Joe, and his blood brother, Snake Eyes.', ' Storm Shadow is one of the most prominent characters in the G.I. Joe: A Real American Hero franchise, and has appeared in every series since its inception.', ' He is portrayed by Lee Byung-hun in the 2009 live-action film \"\", and the 2013 sequel \"\".']], ['Ugolki', ['Ugolki is a two-player board game, similar to halma, that is typically played on an 8×8 grid board with 16 game pieces per player.', ' It is said to have been invented in Europe in the late 18th century.', ' Variations on the size of the board and the number of game pieces also exist.']], ['Craps', ['Craps is a dice game in which the players make wagers on the outcome of the roll, or a series of rolls, of a pair of dice.', ' Players may wager money against each other (playing \"street craps\", also known as \"shooting dice\" or \"rolling dice\") or a bank (playing \"casino craps\", also known as \"table craps\", or often just \"craps\").', ' Because it requires little equipment, \"street craps\" can be played in informal settings.']], ['Connect 4x4', ['Connect 4x4 (spoken as Connect Four by Four) is a three-dimensional-thinking strategy game first released in 2009 by Milton Bradley.', ' The goal of the game is identical to that of its similarly named predecessor, Connect Four.', ' Players take turns placing game pieces in the grid-like, vertically suspended playing field until one player has four of his or her color lined up horizontally, vertically, or diagonally.', ' Unlike its predecessor, Connect 4x4 uses a double grid, two different types of game pieces, and can be played by up to four people at once.']], ['Operation Snake Eyes', ['Operation Snake Eyes was a proposed military operation of the Laotian Civil War.', \" Planned in mid-December 1969 by the U.S. Ambassador to Laos, the planned interdiction of the newly constructed Chinese Road, Route 46, was aimed at halting the road's progress toward the border with Thailand.\", ' The offensive by guerrilla raiders was delayed six months for operational reasons.', ' When it was finally ready to be launched, it was pre-empted by the furor caused by the Cambodian Incursion.', ' Fearful that Operation Snake Eyes would arouse even greater publicity, the Central Intelligence Agency handlers of the guerrillas canceled the operation on orders of the White House.', ' Attempts to limit Chinese expansion toward the south would be left to future operations, such as Operation Phalat and Operation Sourisak Montry.']], ['Snake eyes', ['In gambling in general and the game of Craps in particular, snake eyes is the outcome of rolling the dice in a game and getting only one pip on each die.', ' The pair of pips resembles a pair of eyes, which is appended to the word \"snake\" because of the creature\\'s long-standing association with treachery and betrayal.', ' The dictionary of etymology traces use of the term as far back as 1929.', ' Ancient Roman dice games used the term \"dogs\" to describe a throw of double ones, referring to this as \"the dog throw\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.783\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7d97205542990b8f5039c7', 'answer': 'Ralph Edmund Stanley', 'question': 'What member from The Clinch Mountain Boys plays at the Huck Finn Jubilee Bluegrass Music Festival?', 'supporting_facts': [['Huck Finn Jubilee', 3], ['Ralph Stanley', 0], ['Ralph Stanley', 1]], 'context': [['Vernon Crawford &quot;Jack&quot; Cooke', ['Vernon Crawford (Jack) Cooke (December 6, 1936 – December 1, 2009) was a bluegrass music vocalist and instrumentalist, known for playing the guitar and bass with artists such as Bill Monroe and Ralph Stanley and the Clinch Mountain Boys.', ' He was one of nine siblings (four brothers and four sisters) and was a native of Norton, Virginia.']], ['Ralph Stanley', ['Ralph Edmund Stanley (February 25, 1927 – June 23, 2016), also known as Dr. Ralph Stanley, was an American bluegrass artist, known for his distinctive singing and banjo playing.', ' Stanley began playing music in 1946, originally with his brother Carter as part of The Stanley Brothers, and most often as the leader of his band, The Clinch Mountain Boys.']], ['James Alan Shelton', ['James Alan Shelton (November 3, 1960 – June 3, 2014) was an American bluegrass guitarist.', ' Shelton was a solo musician, released 10 albums, and performed with the Clinch Mountain Boys.']], ['Leslie Keith', ['Leslie Keith (March 30, 1906 – December 28, 1977) was an American bluegrass musician.', ' Known as a formidable fiddler who won many contests, Keith once played with Ralph Stanley and the Clinch Mountain Boys, as well as The Stanley Brothers.', ' He is best known for the tune he put together out of pieces of older tunes, \"Black Mountain Rag\".']], ['The Stanley Brothers', ['The Stanley Brothers were an American bluegrass duo made up of brothers Carter Stanley (1925–1966) and Ralph Stanley (1927–2016).', ' Ralph and Carter performed as The Stanley Brothers with their band, The Clinch Mountain Boys, from 1946 to 1966.', \" Ralph kept the band name when he continued as a solo after Carter's death, from 1967 until his own death in 2016.\"]], ['Carter Stanley', ['Carter Glen Stanley (August 27, 1925 – December 1, 1966) was a bluegrass music lead singer, songwriter, and rhythm guitar player.', ' He formed \"The Stanley Brothers and The Clinch Mountain Boys\" band together with his brother Ralph.', ' The Stanley Brothers are generally acknowledged as the first band after Bill Monroe & the Blue Grass Boys to play in the bluegrass genre.', ' According to some historians, their recording of \"Molly and Tenbrooks\" (aka \"The Racehorse Song\") marked the beginning of bluegrass as a genre.']], ['Josh Graves', ['Josh Graves (September 27, 1927 Tellico Plains, Monroe County, Tennessee – September 30, 2006), born Burkett Howard Graves, was an American bluegrass musician.', ' Also known by the nicknames \"Buck,\" and \"Uncle Josh,\" he is credited with introducing the resonator guitar (commonly known under the trade name of Dobro) into bluegrass music shortly after joining Lester Flatt, Earl Scruggs and the Foggy Mountain Boys in 1955.', ' He was inducted into the International Bluegrass Music Hall of Honor in 1977.']], ['George Shuffler', ['George Shuffler (April 11, 1925 – April 7, 2014) was an American bluegrass guitar player and an early practitioner of the crosspicking style.', \" During his career Shuffler played with The Bailey Brothers, The Stanley Brothers and Ralph Stanley's Clinch Mountain Boys.\", ' He was a 2007 recipient of the North Carolina Heritage Award and in 2011 was elected to the International Bluegrass Music Hall of Fame.']], ['Johnson Mountain Boys', ['The Johnson Mountain Boys were a popular bluegrass band throughout the 1980s from the Washington, D.C. area.', ' Their style favored a more traditional approach to bluegrass than some of their contemporaries.', ' They released ten albums and toured widely, playing venues such as Madison Square Garden, The White House, the Lincoln Center and the Grand Ole Opry in the United States.', ' Other tours took them around the world to England, Japan, and Africa.', ' The group was frequently recognized with nominations for Grammy Awards, International Bluegrass Music Awards, and awards from the Society for the Preservation of Bluegrass Music in America.', \" Many of the band's members are still active in other musical groups and settings.\"]], ['Huck Finn Jubilee', ['The Huck Finn Jubilee Bluegrass Music Festival or simply The Huck Finn Jubilee is an annual three-day bluegrass event held in Ontario, California.', ' It is held during the second weekend in June at the Cucamonga-Guasti Regional Park and features RV and tent camping, traditional \"Americana\" activities, and live bluegrass music.', ' The festival also features activities, such as greased pole climbing, tomahawk throwing, pie eating, river rafting, and arts and crafts.', ' The event is also the West Coast’s biggest bluegrass festival, with acts such as The String Cheese Incident, Ralph Stanley, Steve Martin & the Steep Canyon Rangers, and Rhonda Vincent.', ' On September 30, 2016 the Greater Ontario Convention & Visitors Bureau announced that 2016 would be the last festival sponsored by it at the Cucamonga-Guasti Regional Park.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.784\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ade9545554299728e26c741', 'answer': 'Swiss made', 'question': 'What is special about the wristwatches that Favre-Leuba manufactures?', 'supporting_facts': [['Favre-Leuba', 0], ['Swiss made', 0]], 'context': [['Muramatsu Flutes', ['The Muramatsu company is a Japanese company that manufactures flutes.', ' Their handmade flutes are made from sterling silver, 9K, 14K, 18K, and 24K gold, as well as platinum.', ' The 18K, 24K, and platinum flutes may be purchased by special order only.']], ['Kimber Manufacturing', ['Kimber Manufacturing is an American company that designs, manufactures, and distributes small arms such as M1911 pistols, Solo pistols and rifles.', ' The USA Shooting Team, Marines assigned to Special Operations Command, and the LAPD SWAT team use Kimber pistols.']], ['Backlight', ['A backlight is a form of illumination used in liquid crystal displays (LCDs).', ' As LCDs do not produce light by themselves (unlike, for example cathode ray tube (CRT) displays), they need illumination (ambient light or a special light source) to produce a visible image.', ' Backlights illuminate the LCD from the side or back of the display panel, unlike frontlights, which are placed in front of the LCD.', ' Backlights are used in small displays to increase readability in low light conditions such as in wristwatches, and are used in smart phones, computer displays and LCD televisions to produce light in a manner similar to a CRT display.', ' A review of some early backlighting schemes for LCDs is given in a report \"Engineering and Technology History\" by Peter J. Wild.']], ['Favre-Leuba', ['Favre-Leuba is a Swiss manufacturer of wristwatches headquartered in Solothurn, Switzerland.', ' It was a pioneer in watch design, manufacturing and distribution, thus contributing immensely to the Swiss watchmaking industry.', ' The foundation of the brand was laid in 1737 when Abraham Favre was registered as a watchmaker, so it has been reported as the second-oldest watch brand in Switzerland.']], ['Watts Brothers Tool Works', ['Watts Brothers Tool Works is a tool manufacturer located in Wilmerding, Pennsylvania.', ' They are known for manufacturing drill bits that can drill square holes, including blind holes which cannot be made with other methods such as broaching.', ' The Harry Watts square drill bit is based on a Reuleaux triangle shape, and is used together with a guide and a special chuck to make a square hole.', ' Similarly, the company also manufactures drill bits for other angular holes such as pentagons and hexagons.']], ['Ariella Fashion House', ['Ariella is a British fashion brand of cocktail, evening and special occasion wear founded in 1966.', ' Ariella designs, manufactures, wholesales and retails women’s fashion.', ' Ariella sells under their own labels - retail label Ariella London and designer label Ariella Couture, as well as under clients’ labels.', ' In April 2015 Ariella opened its flagship store in Brent Cross Shopping Centre.']], ['Lavet type stepping motor', ['The Lavet type stepping motor has widespread use as a drive in electro-mechanical clocks and is a special kind of single-phase stepping motor.', ' Both analog and stepped-movement quartz clocks use the Lavet type stepping motor.', ' See Quartz clock.', ' Through miniaturization it can be used in wristwatches and requires very little power, making a battery last for many years.', ' The French engineer Marius Lavet is known as the inventor for this kind of drives and described it in 1936 in his patent application FR823395.']], ['Triangle Group', ['Triangle Group (also known as Triangle Tyre) is a Chinese tire company that manufactures a range of tires for vehicles from passenger cars to construction equipment and tires fit for special purposes.', ' As of 2015 it is the 14th largest tire maker in the world according to Tyres & Accessories.']], ['Carpenter Technology Corporation', ['Carpenter Technology Corporation develops, manufactures and distributes cast/wrought and powder metal stainless steels and special alloys including high temperature (iron-nickel-cobalt base), stainless, superior corrosion resistant, controlled expansion alloys, ultra-high strength and implantable alloys, tool and die steels and other specialty metals, as well as cast/wrought titanium alloys.', ' It also manufactures and rents down-hole drilling tools and components used in the oil and gas industry.']], ['Swiss made', ['Swiss made is a label used to indicate that a product was made in Switzerland.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.784\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab3b42c5542992ade7c6e4f', 'answer': 'Chicago, IL', 'question': 'What city did the musician whose debut album shares its title with the 1959 Alfred Hitchcock hail from?', 'supporting_facts': [['SoulStice', 0], ['SoulStice', 2], ['North by Northwest', 0]], 'context': [['Alfred Hitchcock filmography', ['Alfred Hitchcock (1899–1980) was an English director and filmmaker.', ' Popularly known as the \"Master of Suspense\" for his use of innovative film techniques in thrillers, Hitchcock started his career in the British film industry as a title designer and art director for a number of silent films during the early 1920s.', ' His directorial debut was the 1925 release \"The Pleasure Garden\".', ' Hitchcock followed this with \"\", his first commercial and critical success.', ' It featured many of the thematic elements his films would be known for such as an innocent man on the run.', ' It also featured the first of his famous cameo appearances.', ' Two years later he directed \"Blackmail\" (1929) which was his first sound film.', ' In 1935 Hitchcock directed \"The 39 Steps\".', ' Three years later he directed \"The Lady Vanishes\" starring Margaret Lockwood, and Michael Redgrave.']], ['Cindy Bernard', ['Cindy Bernard is a Los-Angeles based artist whose artistic practice comprises photography, video, performance and activism.', ' In 2002, Cindy Bernard founded the Society for the Activation of Social Space through Art and Sound(SASSAS), which presents site-relational experimental music.', ' Her numerous Hitchcock references have been discussed in Dan Auiler\\'s \"Vertigo: The Making of a Hitchcock Classic\" (1998), essays by Douglas Cunningham and Christine Spengler in \"The San Francisco of Alfred Hitchcock’s Vertigo: Place, Pilgrimage and Commemoration\" (2012) and Spengler\\'s \"Hitchcock and Contemporary Art\" (2014).']], ['The Girl (2012 TV film)', ['The Girl is a 2012 British television film directed by Julian Jarrold, written by Gwyneth Hughes and produced by the BBC and HBO Films.', ' The film stars Sienna Miller as Tippi Hedren and Toby Jones as Alfred Hitchcock.', ' It is based on Donald Spoto\\'s 2009 book, \"Spellbound by Beauty: Alfred Hitchcock and His Leading Ladies\", which discusses British-born film director Hitchcock and the women who played leading roles in his films.', ' \"The Girl\"\\' s title was inspired by Hitchcock\\'s alleged nickname for Hedren.']], ['Gordon Harker', ['Gordon Harker (7 August 1885 – 2 March 1967) was an English stage and film actor.', ' he had a long career on the stage, from 1902 to the 1950s.', ' One of the last plays he starred in was \"Small Hotel\", a popular comedy he toured in 1955.', ' In addition, he appeared in 68 films between 1921 and 1959, including three silent films directed by Alfred Hitchcock and in several scenes in \"Elstree Calling\" (1930), a revue film co-directed by Hitchcock.', ' He was known for his performance as Inspector Hornleigh in a trilogy of films produced between 1938 and 1940, as well in \"Saloon Bar\" (1940), based on a stage play he had starred in and another one of his stage successes The Poltergeist made into the film \"Things Happen at Night\" (1947), a poltergeist comedy he co-starred in with Alfred Drayton and Robertson Hare.', ' His last major screen role was as the wiley waiter Albert in the 1957 motion picture version of \"Small Hotel\"']], ['Noah Georgeson', ['Noah Georgeson is a Grammy winning musician, producer, and solo recording artist.', ' Georgeson\\'s debut album \"Find Shelter\" was released through Plain Recordings on November 28, 2006.', ' Born in San Anselmo, California, he moved with his family to Nevada City, California at the age of three.', ' Georgeson studied classical guitar and music composition, receiving his BA in composition from San Francisco State University in 2001, and, with a recommendation from Terry Riley, he attended Mills College, receiving his Master of Fine Arts in 2003.', ' While at Mills, Georgeson studied with Fred Frith, Pauline Oliveros, and Alvin Curran.', ' Georgeson first found popular success as a part of San Francisco band The Pleased, along with fellow member Joanna Newsom, whose debut album \"The Milk-Eyed Mender\" he produced.', ' As a musician, producer, and mixer, Georgeson has since worked with Devendra Banhart, The Strokes, Little Joy, Bert Jansch, Charlotte Gainsbourg, Robin Pecknold, Mason Jennings, Cedric Bixler, Adam Green, Os Mutantes, Adan Jodorowsky, Harper Simon, Flo Morrissey, Cate Le Bon, and Rodrigo Amarante.']], ['North by Northwest', ['North by Northwest is a 1959 American thriller directed by Alfred Hitchcock and starring Cary Grant, Eva Marie Saint and James Mason.', ' The screenplay was by Ernest Lehman, who wanted to write \"the Hitchcock picture to end all Hitchcock pictures\".']], ['SoulStice', ['Ashley J. Llorens (also known as SoulStice), was born in 1979 in Chicago, IL.', ' He started seriously pursuing music while earning his B.S. and M.S. at the University of Illinois in Urbana-Champaign.', ' In 2003, SoulStice founded the independent label, Wandering Soul Records, concurrently with the release of his first album, North by Northwest.', ' In addition to his career in music, he continues to pursue his career in Electrical Engineering as a Senior Researcher at the Johns Hopkins University Applied Physics Laboratory.']], ['Holly Dolly', ['Holly Dolly is an animated pop musician whose debut single \"Dolly Song (Ievan Polkka)\" was internationally successful in the Summer of 2006.', ' Holly Dolly is an animated, singing female donkey from Italy.']], ['List of The Alfred Hitchcock Hour episodes', ['The following is a list of the 93 episodes of the television program The Alfred Hitchcock Hour, which is a continuation of the program \"Alfred Hitchcock Presents\" (1955–60).', ' \"The Alfred Hitchcock Hour\", like its predecessor, is an anthology series in the thriller genre.', ' NBC renamed the program when they extended its running-time from about 25 minutes to about 50 minutes.', ' Both programs were hosted by Alfred Hitchcock, whose directorial work in thriller films is extremely influential.', ' Hitchcock directed only one episode of \"The Alfred Hitchcock Hour\": \"I Saw The Whole Thing\" (Season 1, episode 4).']], ['George Tomasini', ['George Tomasini (April 20, 1909 – November 22, 1964) was an American film editor, born in Springfield, Massachusetts who had a decade long collaboration with director Alfred Hitchcock, editing nine of his movies between 1954-1964.', ' Tomasini edited many of Hitchcock\\'s best-known works, such as \"Rear Window\" (1954), \"Vertigo\" (1958), \"North by Northwest\" (1959), \"Psycho\" (1960), and \"The Birds\" (1963), as well as other well-received films such as \"Cape Fear\" (1962).', ' On a 2012 listing of the 75 best edited films of all time, compiled by the Motion Picture Editors Guild based on a survey of its members, four films edited by Tomasini for Hitchcock appear.', ' No other editor appeared more than three times on this listing.', ' The listed films were \"Psycho\", \"Vertigo\", \"Rear Window\", and \"North by Northwest\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.785\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae1388e5542997b2ef7d13a', 'answer': 'more than two decades', 'question': 'Double Jeopardy is a 1999 film starring an American political activist best known for an ongoing acting career spanning how long?', 'supporting_facts': [['Double Jeopardy (1999 film)', 0], ['Ashley Judd', 0], ['Ashley Judd', 1], ['Ashley Judd', 2]], 'context': [['It Is the Law', ['It Is the Law is a 1924 American silent mystery film directed by J. Gordon Edwards and starring Arthur Hohl, Herbert Hayes, and Mona Palma.', ' It is a film adaptation of the eponymous 1922 Broadway play by Elmer Rice, itself based on a novel by Hayden Talbot.', ' The film depicts the story of Ruth Allen (Palma), who marries Justin Victor (Heyes) over competing suitor Albert Woodruff (Hohl).', ' Seeking revenge for this slight, Woodruff fakes his death by killing a drifter who resembles him, and frames Victor for the murder.', ' When Woodruff returns to court Allen under a new identity, she sees through his disguise.', ' Once Victor is freed from prison, he kills Woodruff, and goes free because a conviction would constitute double jeopardy.']], ['Frank Meyer (political philosopher)', ['Frank Straus Meyer (1909–1972) was an American philosopher and political activist best known for his theory of \"fusionism\" – a political philosophy that unites elements of libertarianism and traditionalism into a philosophical synthesis which is posited as the definition of modern American conservatism.', ' Meyer\\'s philosophy was presented in two books, primarily \"In Defense of Freedom: A Conservative Credo\" (1962) and also in a collection of his essays, \"The Conservative Mainstream\" (1969).', ' Fusionism has been summed up by E. J. Dionne, Jr. as “utilizing libertarian means in a conservative society for traditionalist ends.”']], ['Béatrice Picard', ['Béatrice Picard, {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} , {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} (born July 3, 1929 in Montreal, Quebec) is a Canadian actress.', ' She is well known in Quebec for the countless roles she has played on the French Canadian theatre and television scene during an ongoing acting career spanning over six decades.', ' She became a household name in Quebec for her acting role as Angelina Desmarais in one of the first French Canadian \"télé-roman\" series called \"Le survenant\" in the early days of French-speaking television.', ' She then went on to a prolific career in televised comedies such as \"Cré Basil\" and \"Symphorien\".', ' She also played in numerous theatre productions, summer plays, and films.', ' Most recently, she is well known as the Quebec French voice of Marge Simpson in \"The Simpsons\".']], ['Douglas S. Cook', ['Douglas S. Cook (1958 – July 19, 2015) was an American screenwriter, known for writing 1996\\'s film \"The Rock\".', ' His other credits included \"Payoff\", \"Holy Matrimony\", \"Double Jeopardy\" and \"Criminal\".', ' Cook wrote all of his screenplays along with his writing partner David Weisberg.', ' They also wrote another action thriller script \"Blank Slate\", which is currently un-produced at Bold Films.']], [\"Scorpion's Revenge\", [\"Scorpion's Revenge (also known as Sasori in U.S.A.) is a 1997 Japanese women in prison film directed by Daisuke Goto, and starring Yoko Saito, Shizuka Ochi and Tetta Sugimoto.\", ' The film was a Japanese/American co-production and was mostly filmed in Los Angeles, California.', ' The plot of the film centers on a woman framed for the murder of her husband, bearing the similar plot to \"Double Jeopardy\" (which was released two years later).']], ['Christopher R. Barron', ['Christopher R. Barron (born December 15, 1973) is an American political activist best known as the cofounder of GOProud, a political organization representing gay conservatives.', \" He is the president of CapSouth Consulting, a political consulting firm, and previously the national political director for Log Cabin Republicans, where he directed the organization's federal lobbying efforts and media relations.\"]], ['Double Jeopardy (1999 film)', ['Double Jeopardy is a 1999 American neo noir adventure crime thriller film directed by Bruce Beresford and starring Tommy Lee Jones, Ashley Judd, and Bruce Greenwood.', ' The film is about a woman wrongfully imprisoned for murder who tracks down her husband who had framed her while eluding her parole officer.']], ['Ashley Judd', ['Ashley Judd (born Ashley Tyler Ciminella; April 19, 1968) is an American actress and political activist.', ' She grew up in a family of successful performing artists as the daughter of country music singer Naomi Judd and the sister of Wynonna Judd.', ' While she is best known for an ongoing acting career spanning more than two decades, she has increasingly become involved in global humanitarian efforts and political activism.']], ['David Weisberg', ['David Weisberg is an American screenwriter, best known for writing 1996 film \"The Rock\", which he co-wrote with his writing partner Douglas Cook, who died on July 19, 2015.', ' His other credits with Cook include \"Payoff\", \"Holy Matrimony\", \"Double Jeopardy\" and \"Criminal\".', ' They also wrote another action thriller script \"Blank Slate\", which is currently un-produced at Bold Films.']], ['Double Jeopardy (1955 film)', ['Double Jeopardy American crime film directed by R.G. Springsteen and starring Rod Cameron, Gale Robbins and Allison Hayes.', ' It is also known by the alternative title of Crooked Ring.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.786\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7560585542992d0ec05f76', 'answer': 'Maurice Ravel', 'question': \"Arbie Orenstein is known as a scholar on the life of what composer internationally regarded as France's greatest living composer? \", 'supporting_facts': [['Arbie Orenstein', 0], ['Maurice Ravel', 2]], 'context': [['Veljo Tormis', ['Veljo Tormis (7 August 1930 – 21 January 2017) was an Estonian composer, regarded as one of the greatest living choral composers and one of the most important composers of the 20th century in Estonia.', ' Internationally, his fame arises chiefly from his extensive body of choral music, which exceeds 500 individual choral songs, most of it a cappella.', ' The great majority of these pieces are based on traditional ancient Estonian folksongs (\"regilaulud\"), either textually, melodically, or merely stylistically.']], ['Theodor Mommsen', ['Christian Matthias Theodor Mommsen (30 November 1817 – 1 November 1903) was a German classical scholar, historian, jurist, journalist, politician, archaeologist and writer generally regarded as one of the greatest classicists of the 19th century.', ' His work regarding Roman history is still of fundamental importance for contemporary research.', ' He received the Nobel Prize in Literature in 1902 for being \"the greatest living master of the art of historical writing, with special reference to his monumental work \"A History of Rome\"\", after having been nominated by 18 members of the Prussian Academy of Sciences.', ' He was also a prominent German politician, as a member of the Prussian and German parliaments.', ' His works on Roman law and on the law of obligations had a significant impact on the German civil code (BGB).']], ['Maurice Ravel', ['Joseph Maurice Ravel (] ; 7 March 1875 – 28 December 1937) was a French composer, pianist and conductor.', ' He is often associated with impressionism along with his elder contemporary Claude Debussy, although both composers rejected the term.', \" In the 1920s and 1930s Ravel was internationally regarded as France's greatest living composer.\"]], ['Marius Trésor', ['Marius Trésor (born 15 January 1950 in Sainte-Anne, Guadeloupe) is a retired football defender from France, who was named by Pelé as one of the top 125 greatest living footballers.', \" He is considered as one of the France's best defenders of all time, and he is regarded as one of France's greatest ever players.\"]], ['Pearl Chertok', ['Pearl Chertok (June 18, 1918, in Laconia, New Hampshire – August 1, 1981 in White Plains, New York) was an internationally regarded harpist and composer for harp.']], ['Arbie Orenstein', ['Arbie Orenstein (born 1937) is an American musicologist, author, academic and pianist, known as a scholar of the life and works of the composer Maurice Ravel and, more generally, as an expert on Jewish music.']], ['Stefans Grové', ['Stefans Grové (born 23 July 1922, Bethlehem, Orange Free State, South Africa; – 29 May 2014, Pretoria) was a South African composer.', ' Before his death the following assessment was made of him: \"He is regarded by many as Africa\\'s greatest living composer, possesses one of the most distinctive compositional voices of our time\".']], ['Javier Torres Maldonado', ['Javier Torres Maldonado (born 1968) is a Mexican-Italian composer internationally recognized for, mostly, his orchestral, chamber, vocal and electro-acoustic works.']], [\"Beethoven's musical style\", ['Ludwig van Beethoven is universally viewed as one of the most influential figures in the history of classical music.', ' Since his lifetime, when he was \"universally accepted as the greatest living composer\", Beethoven\\'s music has remained among the most performed, discussed and reviewed.', ' Scholarly journals are devoted to analysis of his life and work.', ' He has been the subject of numerous biographies and monographs, and his music was the driving force behind the development of Schenkerian analysis.', ' He is widely considered as among the most important composers, and along with Bach and Mozart, his music is the most frequently recorded.']], ['Krzysztof Penderecki', ['Krzysztof Eugeniusz Penderecki ( ; ] ; born 23 November 1933) is a Polish composer and conductor. \"', 'The Guardian\" has called him Poland\\'s greatest living composer.', ' Among his best known works are his \"Threnody to the Victims of Hiroshima\", \"Symphony No. 3\", \"St. Luke Passion\", \"Polish Requiem\", \"Anaklasis\", \"Utrenja\", four operas, eight symphonies and other orchestral pieces, a variety of instrumental concertos, choral settings of mainly religious texts, as well as chamber and instrumental works.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.787\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab626d555429953192ad279', 'answer': 'lower Manhattan', 'question': 'Anthony Avent played basketball fo a High School that is located in a city approcimately 8 mi west of where?', 'supporting_facts': [['Anthony Avent', 0], ['Anthony Avent', 3], ['Newark, New Jersey', 0], ['Newark, New Jersey', 3]], 'context': [['Anthony Avent', ['Anthony Avent (born October 18, 1969) is a retired American professional basketball player who was selected by the Atlanta Hawks in the first round (15th pick overall) of the 1991 NBA Draft.', ' Born in Rocky Mount, North Carolina, Avent played for the Milwaukee Bucks, Orlando Magic, Vancouver Grizzlies, Utah Jazz and Los Angeles Clippers in six NBA seasons.', ' He played collegiately at Seton Hall University where he played in the 1989 NCAA championship game.', ' Prior to Seton Hall, Avent played at Malcolm X Shabazz High School in Newark, New Jersey.']], ['James Blackmon Sr.', ['James Blackmon Sr. (born August 7, 1964) is an American basketball coach and former noted college and high school player.', ' Blackmon has coached two teams to the class 2A high school basketball championship in Indiana, winning back-to-back titles in 2008 and 2009.', ' Blackmon played basketball in college for the University of Kentucky Wildcats.', ' As a senior at Marion High School, in Marion, Indiana, Blackmon was runner-up for the award of Indiana Mr. Basketball, won that year by Steve Alford.', \" Blackmon was named to the McDonald's All-American and Parade All-American teams in recognition of his high school success.\"]], ['Finchampstead', ['Finchampstead is a village and civil parish in the Wokingham Borough of Berkshire, England.', ' Its northern extremity is 2 mi south of Wokingham, 5 mi west of Bracknell, 8 mi south-east of Reading, and 33 mi west of Central London.', \" It is an affluent area, with the village ranking as Britain's 31st wealthiest.\", ' It has a high standard of living and is rated as one of the most desirable places to live in the UK.']], ['Presley Askew', ['Presley Askew (November 17, 1909 – February 7, 1994) was an American basketball and baseball coach.', ' Overall Askew won 169 games at New Mexico State and Arkansas and had an overall record of 509–312 in all high school and college coaching.', ' Born in Red Oak, Oklahoma, Askew played basketball and graduated from Red Oak High School in 1926.', ' He played for and graduated from Eastern Oklahoma State College, and eventually Oklahoma State University in 1930.', ' He began coaching at Fanshawe Public School and became head varsity coach in 1932.', ' In 1937 Askew moved to his hometown Red Oak High School to coach and was there until 1942 when he moved on to Van Buren High School in Arkansas.', \" Askew's teams at Van Buren were very competitive and went to the state championship tournaments.\"]], ['Weldon Drew', [\"Weldon Drew (born April 22, 1935) was the head men's basketball coach at New Mexico State University from 1979 to 1985.\", ' He was named to the position in 1979 as the successor to Ken Hayes who left to become head coach at Oral Roberts University.', ' Drew was previously an assistant coach for New Mexico State University from 1975 to 1979.', \" Drew came to NMSU after coaching high school basketball for Houston's Kashmere High School (485-135 record in 18 seasons), where he left with a 78-game winning streak after winning two consecutive Texas 4A state championships and the high school national championship.\", ' Drew also won national coach of the year in 1975.', \" The NMSU job was Drew's first head coaching position at the college level.\", ' Drew was the 20th person to hold the head coaching position in the Aggie basketball history.', ' After a dismal 1984-85 season, Drew was fired.', ' He then went to be an assistant coach at Oklahoma State for two seasons.', ' In 1987 Drew became the head coach at traditionally-black Langston University in Oklahoma.', ' Drew graduated from Fisk University in 1957 after a standout career playing basketball.', ' Drew graduated high school and played basketball at Wheatley High School in Houston.']], ['Newark, New Jersey', ['Newark ( or also locally ) is the most populous city in the U.S. state of New Jersey and the seat of Essex County.', \" As one of the nation's major air, shipping, and rail hubs, the city had a population of 277,140 in 2010, making it the nation's 67th most-populous municipality, after being ranked 63rd in the nation in 2000.\", \" For 2015, the Census Bureau's Population Estimates Program calculated a population of 281,944, an increase of 1.7% from the 2010 enumeration, ranking the city the 70th largest in the nation.\", ' Newark is the second largest city in the New York metropolitan area, located approximately 8 mi west of lower Manhattan.']], ['Larry Friend', ['Larry Haskell Friend (April 14, 1935 – February 27, 1998) was an American National Basketball Association (NBA) player.', ' Friend was born and raised in Chicago, Illinois and played basketball at Marshall High School in Chicago.', ' However, he moved to Los Angeles, California before his senior year and played basketball at Fairfax High School.', ' Friend first played college basketball at Los Angeles City College, where he was named an All-American Junior College.', ' He then transferred to the University of California, where he was a three-year starter.', ' He averaged 19.1 points per game his senior season and was also named to the AP All-American third team.', ' Friend was drafted with the fifth pick in the second round of the 1957 NBA Draft by the New York Knicks.', ' In his one season with the Knicks, Friend averaged 4.0 points, 2.4 rebounds, and 1.1 assists per game.', ' In 1961-62 Friend returned to professional basketball to play for the Los Angeles Jets in the American Basketball League.', ' He appeared in thirty-nine games for the Jets and averaged 11.0 points and 3.7 rebounds per game, while also leading the league in three-point shooting (58-163).', ' Due to financial problems, the Jets folded midway through their first season.', ' Following his playing career, Friend owned an investment business.', ' He died on February 27, 1998 in Newport, California of prostate cancer.']], ['Canton, Michigan', ['Canton is a charter township of Wayne County in the U.S. state of Michigan.', ' It is located about 8 mi west of the city limits of Detroit and 8 mi east of the city limits of Ann Arbor.', \" As of the 2010 census, the township had a population of 90,173, making it Michigan's second largest township and eleventh largest community.\", ' Canton is ranked as 96th highest-income place in the United States with a population of 50,000 or more.', ' Canton Township is also consistently ranked as one of the safest cities in the United States, as well as in the state of Michigan.', ' In 2015, the township was ranked as the 29th safest city in America.', \" Canton is one of Michigan's fastest growing communities.\"]], ['Kelsall', ['Kelsall is a medium-sized agricultural/commuter village and civil parish in the unitary authority of Cheshire West and Chester and the ceremonial county of Cheshire, England.', ' It is located around 8 mi east of Chester, 8 mi west of Northwich and 4 mi north west of Tarporley.', ' The village is situated on Kelsall Hill, a part of the Mid-Cheshire Ridge, the broken line of sandstone hills that divide the west Cheshire Plain from its eastern counterpart.', ' The ridge includes other hills including Peckforton, Beeston, Frodsham and Helsby.']], ['Jeff Horner', ['Jeff Horner (born August 1, 1983) is an American former basketball player and current assistant coach for the University of North Dakota.', ' He is best known as a point guard for the University of Iowa Hawkeyes basketball team.', ' Horner currently holds the University of Iowa career three point record at 232 shots made.', ' Horner was born in Mason City, Iowa.', ' His father Bob Horner was Mason City High School varsity basketball coach.', ' Horner made a verbal commitment to the University of Iowa while in the ninth grade.', ' Horner was the head varsity basketball coach at Valley High School in West Des Moines, Iowa from 2010-2014.', \" Horner's sister Kristin played basketball for Drake University.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.788\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8e1027554299653c1aa15f', 'answer': '2009 Big 12 Conference', 'question': 'Which year and which conference was the 14th season for this conference as part of the NCAA Division that the Colorado Buffaloes played in with a record of 2-6 in conference play?', 'supporting_facts': [['2009 Colorado Buffaloes football team', 0], ['2009 Colorado Buffaloes football team', 2], ['2009 Big 12 Conference football season', 0]], 'context': [[\"2016–17 Colorado Buffaloes men's basketball team\", [\"The 2016–17 Colorado Buffaloes men's basketball team represented the University of Colorado in the 2016–17 NCAA Division I men's basketball season.\", ' They were led by head coach Tad Boyle in his seventh season at Colorado.', ' The Buffaloes played their home games at Coors Events Center in Boulder, Colorado as members of the Pac-12 Conference.', ' They finished the season 19–15, 8–10 in Pac-12 play to finish in seventh place.', ' They defeated Washington State in the first round of the Pac-12 Tournament to advance to the quarterfinals where they lost to Arizona.', ' They were invited to the National Invitation Tournament where the lost in the first round to UCF.']], [\"2015–16 Colorado Buffaloes men's basketball team\", [\"The 2015–16 Colorado Buffaloes men's basketball team represented the University of Colorado in the 2015–16 NCAA Division I men's basketball season.\", \" This was Tad Boyle's sixth season as head coach at Colorado.\", ' The Buffaloes played their home games at Coors Events Center in Boulder, Colorado and were members of the Pac-12 Conference.', ' They finished the season 22–12, 10–8 in Pac-12 play to finish in fifth place.', ' The defeated Washington State in the first round of the Pac-12 Tournament to advance to the quarterfinals where they lost to Arizona.', ' They received an at-large bid to the NCAA Tournament where they lost in the first round to Connecticut.']], ['2009 Colorado Buffaloes football team', ['The 2009 Colorado Buffaloes football team represented the University of Colorado in the 2009 NCAA Division I FBS college football season.', ' The Buffaloes were led by fourth year head coach Dan Hawkins and played their home games at Folsom Field.', ' The Buffaloes finished the season with a record of 3–9 and 2–6 in Big 12 play.']], ['2015 Colorado Buffaloes football team', ['The 2015 Colorado Buffaloes football team represented the University of Colorado at Boulder during the 2015 NCAA Division I FBS football season.', ' Led by third-year head coach Mike MacIntyre, the Buffaloes played their home games on-campus at Folsom Field in Boulder and were members of the South Division of the Pac-12 Conference.', ' They finished the season 4–9, 1–8 in Pac-12 play to finish in last place in the South Division.']], ['2012 Colorado Buffaloes football team', ['The 2012 Colorado Buffaloes football team represented the University of Colorado Boulder during the 2012 NCAA Division I FBS football season.', ' Led by second-year head coach and alumnus Jon Embree, the Buffaloes played their home games on-campus at Folsom Field in Boulder and were members of the Pac-12 Conference.', ' On November 25, 2012, head coach Jon Embree was fired after compiling a 4–21 record including 1–11 in his final year, the worst year in the history of Colorado Buffaloes football.']], ['1989 Colorado Buffaloes football team', ['The 1989 Colorado Buffaloes football team represented the University of Colorado at Boulder during the 1989 NCAA Division I-A football season.', ' Colorado finished with the most wins in school history, surpassing the 1971 team, and their first conference championship since 1976.', ' The Buffaloes played for the national title but lost to Notre Dame in the Orange Bowl.', ' The team dedicated the season to senior and former starting quarterback Sal Aunese, who was diagnosed with stomach cancer in February and died on September 23 due to complications from the disease.']], ['2011 Colorado Buffaloes football team', ['The 2011 Colorado Buffaloes football team represented the University of Colorado at Boulder in the 2011 NCAA Division I FBS football season.', ' Led by first-year head coach and alumnus Jon Embree, the Buffaloes played their home games on-campus at Folsom Field in Boulder and were first-year members of the newly expanded Pac-12 Conference.', ' They finished with a record of 3–10, 2–7 in Pac-12 play, in a tie for last place in the South Division.']], ['2014 Colorado Buffaloes football team', ['The 2014 Colorado Buffaloes football team represented the University of Colorado at Boulder during the 2014 NCAA Division I FBS football season.', ' Led by second-year head coach Mike MacIntyre, the Buffaloes played their home games on-campus at Folsom Field in Boulder and were members of the South Division of the Pac-12 Conference.', ' They finished the season 2–10, 0–9 in Pac-12 play to finish in last place in the South Division.']], ['2009 Big 12 Conference football season', ['The 2009 Big 12 Conference football season was the 14th season for the Big 12, as part of the 2009 NCAA Division I FBS football season.']], [\"2014–15 Colorado Buffaloes men's basketball team\", [\"The 2014–15 Colorado Buffaloes men's basketball team represented the University of Colorado in the 2014–15 NCAA Division I men's basketball season.\", \" This was Tad Boyle's fifth year as head coach at Colorado.\", ' The Buffaloes played their home games at the Coors Events Center in Boulder, Colorado as members of the Pac-12 Conference.', ' They finished the season 16–18, 7–11 in Pac-12 play to finish in a three way tie for eighth place.', ' They advanced to the quarterfinals of the Pac-12 Tournament where they lost to Oregon.', ' They were invited to the College Basketball Invitational where they defeated Gardner–Webb in the first round before losing in the second round to Seattle.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.789\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae2a9c6554299492dc91c42', 'answer': 'Tumi Holdings, Inc.', 'question': 'Which New Jersey-based manufacturer of suitcases and bags for travel is located in the Shops at Columbus Circle in New York City?', 'supporting_facts': [['The Shops at Columbus Circle', 0], ['The Shops at Columbus Circle', 1], ['Tumi Inc.', 0]], 'context': [['Per Se (restaurant)', ['Per Se is a New American and French restaurant located on the fourth floor of the Time Warner Center at 10 Columbus Circle (at West 60th Street and Broadway) in Manhattan in New York City, owned by chef Thomas Keller.', ' In 2011, it was called the best restaurant in New York City by \"The New York Times\".', ' The chef is Eli Kaimeh.', ' Per Se is currently the third most expensive restaurant in the world after Sublimotion and Urasawa with an average guest spending approximately $851.']], ['Campuses of Fordham University', ['The Campuses of Fordham University are located within New York City and the New York City metropolitan area.', \" The university's original Rose Hill campus is located in The Bronx on Fordham Road, while the Lincoln Center campus is located in Manhattan, one block west of Columbus Circle.\", ' The Westchester campus is located in Harrison, New York in Westchester County.', ' Additionally, Fordham University maintains a study abroad center in the United Kingdom and field offices in Spain and South Africa.']], ['Tumi Inc.', ['Tumi Holdings, Inc., is a South Plainfield, New Jersey-based manufacturer of suitcases and bags for travel.', ' Founded in 1975 by Charlie Clifford after a stint in the Peace Corps in Peru, the company is named after a Peruvian ceremonial knife used for sacrifices.', ' Tumi, Inc. was a unit of Doughty Hanson & Co. from 2004 until after its 2012 initial public offering.']], ['Time Warner Center', ['Time Warner Center is a mixed use (office/commercial and residential) twin-tower building in New York City.', ' Developed by The Related Companies and AREA Property Partners (formerly known as Apollo Real Estate Advisors), its design by David Childs and Mustafa Kemal Abadan of Skidmore, Owings & Merrill, consists of two 750 ft twin towers bridged by a multi-story atrium containing upscale retail shops.', ' Construction began in November 2000, following the demolition of the New York Coliseum, and a topping-out ceremony was held on February 27, 2003.', ' The property had the highest-listed market value in New York City, $1.1 billion, in 2006.', ' Originally constructed as the AOL Time Warner Center, the building encircles the western side of Columbus Circle and straddles the border between Midtown and the Upper West Side.', ' The total floor area of 2.8 e6ft2 is occupied by office space (notably the offices of Time Warner and an R&D Center for VMware), residential condominiums, and the Mandarin Oriental, New York hotel.', ' The Shops at Columbus Circle is an upscale shopping mall located in a curving arcade at the base of the building, with a large Whole Foods Market grocery store on the lower level.']], ['Columbus Circle (Syracuse, New York)', ['Columbus Circle is a neighborhood and plaza in the downtown section of Syracuse, New York.', ' Columbus Monument was designed by the Syracuse-born architect, Dwight James Baum in 1934.', \" Columbus Circle is home to Syracuse's two cathedrals, the Episcopalian St. Paul's Cathedral and the Roman Catholic Cathedral of the Immaculate Conception.\", \" As well as County Court House and the County's John H. Mulroy Civic Center, home of the Onondaga County Government.\"]], ['The Shops at Columbus Circle', ['The Shops at Columbus Circle is an urban shopping mall in the Time Warner Center in Manhattan, New York City — a complex of skyscrapers that was completed in 2003.', ' It is located at Columbus Circle, next to the southwestern corner of Central Park.', \" The shopping mall includes Amazon Books, H&M, L'Occitane, Michael Kors, Hugo Boss, Tumi, Coach, Cole Haan, Thomas Pink, J.Crew and Stuart Weitzman.\", ' The mall also has several restaurants such as the Michelin 3-star Per Se, Masa (allegedly the most expensive restaurant in New York ), the East Coast flagship of Williams-Sonoma, and a Whole Foods Market.', ' It is owned by The Related Companies.']], ['Forman Mills', ['Forman Mills, Inc. is a Pennsauken, New Jersey-based retail chain and department store with 35 stores, located in Philadelphia, Baltimore, Delaware, New Jersey, Washington DC, Chicago, Cleveland, Detroit, New York City and their suburbs.', ' They also operate a store at the Iverson Mall in Hillcrest Heights, Maryland.', ' It was begun by Richard Forman when he started selling items at the Columbus Farmers Market.', ' The chain is known for their low-priced designer clothing such as shirts, pants, shorts, capri pants, and hats.']], ['Chris Doyle (artist)', ['Chris Doyle is a multi-media artist who lives in New York City.', ' His major public projects have included BRIGHT CANYON, presented by the Times Square Alliance (2014); LEAP, presented by Creative Time in Columbus Circle (2000) and Commutable, presented by the Public Art Fund on the Lower East Side (1996), all in New York City.', ' His work has also been shown at The Brooklyn Museum of Art, The Queens Museum of Art, P.S.1 Museum of Contemporary Art, the Kupferstichkabinett Berlin, Germany, and as part of the New York Video Festival at Lincoln Center.', ' In 2015 he created a major immersive sculpture, video and sound piece for Wave Hill Botanical Gardens in New York.']], ['Columbus Circle', ['Columbus Circle, named for Christopher Columbus, is a traffic circle and heavily trafficked intersection in the New York City borough of Manhattan, located at the intersection of Eighth Avenue, Broadway, Central Park South (West 59th Street), and Central Park West, at the southwest corner of Central Park.', ' It is the point from which all official distances from New York City are measured.', ' The name is also used for the neighborhood a few blocks around the circle in each direction.', ' To the south of the circle lies Hell\\'s Kitchen, also known as \"Clinton\", and the Theater District, and to the north is the Upper West Side.']], ['2 Columbus Circle', ['2 Columbus Circle is a 12-story building located on a small, trapezoidal lot on the south side of Columbus Circle on the Upper West Side of Manhattan, New York City.', ' Bordered by 58th Street, 59th Street, Broadway, and Eighth Avenue, it stands on the site of the seven-story Grand Circle Hotel designed by William H. Cauvet.', ' Opened in 1964 after A&P heir Huntington Hartford hired architect Edward Durell Stone to build a museum for him at the site.', \" The building came under controversy in 2002 after the Museum of Arts and Design (MAD) was designated as the building's developer.\", ' MAD subsequently significantly altered its design, including modifying its facade; since 1996, ideas had been put forward for the building to be landmarked, so its proposed landmark status was brought into question with this renovation.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.789\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab53ef1554299488d4d9920', 'answer': 'north', 'question': 'In what directional area of Argentia would one find the Rio Hondo Department?', 'supporting_facts': [['Río Hondo Department', 0], ['Río Hondo Department', 1], ['Santiago del Estero Province', 0]], 'context': [['Rio Hondo bicycle path', ['The Rio Hondo Bike Path is a Class 1 bicycle path that parallels the Rio Hondo (creek) through the San Gabriel Valley, in eastern Los Angeles County, California.', ' The bicycle path is gently graded, and has sections with more greenery and mature trees than other bicycle paths in the area.']], ['Santiago del Estero Province', ['Santiago del Estero (] ), also called \"Santiago\", is a province in the north of Argentina.', ' Neighbouring provinces, clockwise from the north, are Salta, Chaco, Santa Fe, Córdoba, Catamarca and Tucumán.']], ['Rio Hondo (Northern New Mexico)', ['In northern New Mexico, the Rio Hondo begins high in the Sangre de Cristo Mountains near Taos Ski Valley and flows for approximately 20 mi into the Rio Grande.', ' Portions of the Rio Hondo are prized as prime spots for bird-watching and fishing.', ' The river was the subject of a 2005 study by the New Mexico Environment Department Surface Water Quality Bureau into the effects of wastewater from Taos Ski Valley, which is discharged from the Village of Taos Ski Valley Waste Water Treatment Plant.']], ['Rio Hondo (Southern New Mexico)', ['The Rio Hondo is a 79 mi river in southern New Mexico which begins at the confluence of the Rio Bonito and Rio Ruidoso rivers near the town of Hondo, New Mexico.', ' The river flows eastward through the Hondo Valley in the foothills of the Sierra Blanca and Capitan Mountains, roughly paralleling the route of U.S. Route 70 through the towns of Picacho and Tinnie.', ' Near the community of Riverside the river passes through a deep canyon before entering the rolling hills west of Roswell.']], ['Rio Hondo High School', ['Rio Hondo High School is a high school in Rio Hondo, Texas.', ' It is a part of the Rio Hondo Independent School District.']], ['Río Hondo Department', ['Río Hondo Department (Spanish: \"Departamento Río Hondo\" ) is a department of Argentina in Santiago del Estero Province.', ' The capital city of the department is situated in Termas de Río Hondo.']], ['Rio Hondo Preparatory School', ['Rio Hondo Preparatory School, familiarly known as Rio Hondo Prep, RHP or simply Rio, is a day school for grades 6–12.', ' Founded in 1964, it is located in Arcadia, California.']], ['San Román, Orange Walk', ['San Roman village also known as San Roman Rio Hondo is located in the Orange Walk District from the nation of Belize.', ' The village consist of mainly people from Yucatec Maya descent.', ' San Roman Rio Hondo is known for having 100 Year Old Church .']], ['Rio Hondo Independent School District', ['Rio Hondo Independent School District is a public school district based in Rio Hondo, Texas (USA).']], ['Rio Hondo College', ['Rio Hondo College is a community college located in the city of Whittier, California, United States, named after the Rio Hondo.', ' Founded in 1960, it mainly serves the cities of Whittier, Pico Rivera, Santa Fe Springs, El Monte, and South El Monte.', ' Rio Hondo College offers 23 associate degrees for transfer that guarantee transfer to California State Universities, 50 associate degrees and 60 certificates.', ' Rio Hondo offers on-campus, online, and off-campus courses to all of its students.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.789\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abe239355429976d4830a7d', 'answer': 'Brad Silberling', 'question': 'Jamie Harris is best known for his role as The Hook-Handed Man in a movie directed by who ?', 'supporting_facts': [['Jamie Harris (actor)', 0], ['Jamie Harris (actor)', 1], [\"Lemony Snicket's A Series of Unfortunate Events\", 0]], 'context': [['Usman Ally', ['Usman Ally is an American film, stage and television actor.', ' In 2015, Ally won an Obie Award for his role in \"The Invisible Hand\".', ' He has appeared in several stage productions including \"The Elaborate Entrance of Chad Deity\", \"The Jungle Book\" and a production of \"Around the World in 80 Days\".', ' He is known for his on screen roles such as Vincent on \"Agents of S.H.I.E.L.D.\" and The hook-handed man in \"A Series of Unfortunate Events\".']], ['Jamie Harris (actor)', ['Jamie Harris (born May 15, 1963) is a British actor.', ' He is best known for his role as The Hook-Handed Man in \"Lemony Snicket\\'s A Series of Unfortunate Events\", Rodney in \"Rise of the Planet of the Apes\" and Gordon in Marvel\\'s \"Agents of S.H.I.E.L.D.\".']], ['Badla Jatti Da', ['Badla Jatti Da (Punjabi:ਬਦਲਾ ਜੱਟੀ ਦਾ) is a 1991 Punjabi action movie directed by Ravinder Ravi.', ' This movie stars Gugu Gill and Yograj Singh in lead roles.', ' The villain role played by Yograj Singh is considered one of his best.', ' The movie was a blockbuster hit across Punjab.']], [\"Lemony Snicket's A Series of Unfortunate Events\", [\"Lemony Snicket's A Series of Unfortunate Events is a 2004 American dark comedy film directed by Brad Silberling.\", ' It is a film adaptation of \"A Series of Unfortunate Events\" by Lemony Snicket, covering the first three novels \"The Bad Beginning\", \"The Reptile Room\", and \"The Wide Window\".', \" The film stars Jim Carrey, Liam Aiken, Emily Browning, Timothy Spall, Catherine O'Hara, Billy Connolly, Cedric the Entertainer, Luis Guzmán, Jennifer Coolidge and Meryl Streep, as well as Jude Law as the voice of Lemony Snicket.\"]], ['Midnight Sun (2006 film)', ['Song to the Sun, known in Japan as Taiyō no Uta (タイヨウのうた , Song of the Sun ) , is a movie directed by Norihiro Koizumi starring the Japanese artist and singer Yui.', ' In the movie, she plays the role of Kaoru Amane (雨音 薫 \"Amane Kaoru\"), a 16-year-old girl who has the rare skin condition xeroderma pigmentosum (XP), a disease that makes the ultraviolet radiation of sunlight potentially lethal to her.', ' Yui\\'s character is partly based on herself, as she is a singer and guitarist, and she performs three of Yui\\'s songs in the movie; \"It\\'s Happy Line\", \"Good-bye Days\" and \"Skyline\".', ' There has been a 2006 Japanese TV drama starring Takayuki Yamada and Erika Sawajiri, a manga by Bandō Kenji and Minatsuki Tsunami, a 2015 Vietnamese / Japanese drama, and a 2017 American remake, all based on the movie.']], ['Maurice Walsh', ['Maurice Walsh (baptised 23 April 1879 – 18 February 1964) was an Irish novelist best known for the short story \"The Quiet Man\" which was later made into an Oscar-winning movie directed by John Ford and starring John Wayne and Maureen O\\'Hara.', \" He was one of Ireland's best-selling authors in the 1930s.\"]], ['Estelle Hemsley', ['Estelle Hemsley (May 5, 1887 - November 5, 1968) was a prominent early African American actress of stage and screen.', ' She appeared in the stage and screen versions of \"Take a Giant Step\", earning a Golden Globe nomination for Best Supporting Actress in the 1959 movie directed by Philip Leacock.', ' Her other notable film roles include playing Grandmother Topouzoglou in Elia Kazan\\'s 1963 movie \"America, America\" (nominated for the Oscar for Best Picture), the role of Cla-Cla in Mel Ferrer\\'s 1959 film \"Green Mansions\", the mother of Ruby Dee in \"Edge of the City\" (1957), and Catherine in Robert Mulligan\\'s 1965 movie \"Baby the Rain Must Fall\".']], ['Jalam (film)', ['Jalam is a 2016 Malayalam-language movie directed by M. Padmakumar starring Priyanka Nair in the lead role.', \" This is a world's first charity movie, a CSR film by Aries Group directed by M. Padmakumar and produced by Sohan Roy.\", ' Multiple songs from the movie are now in contention for nominations in the Original Song Category for the 88th Academy Awards.', ' ‘Bhoomiyilenganumundo’, ‘Kooduvaykkam’, ‘Yaathra Manoradhamerum’ and ‘Pakalppathichari’ are the songs from Jalam that are competing for the nomination in the category.', ' The movie also vies for nominations in the Best Picture Category at the Oscars.']], ['Man Against the Mob', ['Man Against the Mob (also known as \"Trouble in the City of Angels\") is a 1988 NBC television movie directed by Steven Hilliard Stern, starring George Peppard, Kathryn Harrold and Max Gail.', ' \"Man Against the Mob\" is a precursor of the 2013 theatrical feature \"Gangster Squad\", in that it deals with the post-war formation of a special LAPD unit set up to suppress Organized Crime in Los Angeles.', ' It may have been inspired by the success of the 1987 theatrical feature \"The Untouchables\", a period drama which also depicted an elite law enforcement unit pitted against mobsters.', ' This was designed around the actor George Peppard as a tough LA cop in the late 1940s.', ' A 1989 TV-movie followup, \"Man Against the Mob: The Chinatown Murders\" is a sequel that also stars Peppard.', ' The first movie was a pilot of a proposed NBC series entitled \"City of Angels\" but ended up panning out as only the two TV movies before George Peppard died in 1994.']], ['The Dragon, the Hero', ['The Dragon, The Hero is a Hong Kong martial art movie directed by Godfrey Ho and starring Philip Ko,Dragon Lee, Tino Wong Cheung and Liu Chung-Liang.', ' The movie is considered as one of the best martial arts movie that Godfrey Ho directed outside of the martial arts movie fanbase.', ' The movie is also known as Dragon on Fire.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.790\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7a33205542996a35c1712f', 'answer': 'Province of New York', 'question': 'Out of two American colonies that had a series of skirmishes and raids between 1701 and 1765 at the disputed border, which British proprietary colony became a royal colony on the northeast coast of North America?', 'supporting_facts': [['New York – New Jersey Line War', 0], ['Province of New York', 0]], 'context': [['History of the New Jersey State Constitution', ['Originally, the state of New Jersey was a single British colony, the Province of New Jersey.', ' After the English Civil War, Charles II assigned New Jersey as a proprietary colony to be held jointly by Sir George Carteret and John Berkeley, 1st Baron Berkeley of Stratton.', ' Eventually, the collection of land fees, or quit-rents, from colonists proved inadequate for colonial profitability.', ' Sir George Carteret sold his share of the colony to the Quakers in 1673.', ' Following the sale, the land was divided into East and West Jersey.', ' In 1681, West Jersey adopted a constitution.', ' In 1683, East Jersey adopted one as well.', ' In 1702, the colonies were united again under Anne, Queen of Great Britain, and adopted a constitution in 1776.']], ['Province of Pennsylvania', ['The Province of Pennsylvania, also known as the Pennsylvania Colony, was founded in English North America by William Penn on March 4, 1681 as dictated in a royal charter granted by King Charles II.', ' The name Pennsylvania, which translates roughly as \"Penn\\'s Woods\", was created by combining the Penn surname (in honor of William\\'s father, Admiral Sir William Penn) with the Latin word \"sylvania\", meaning \"forest land.\"', ' The Province of Pennsylvania was one of the two major Restoration colonies, the other being the Province of Carolina.', \" The proprietary colony's charter remained in the hands of the Penn family until the American Revolution, when the Commonwealth of Pennsylvania was created and became one of the original thirteen states.\"]], ['Province of New York', ['The Province of New York (1664–1776) was a British proprietary colony and later royal colony on the northeast coast of North America.', ' As one of the Thirteen Colonies, New York achieved independence and worked with the others to found the United States.']], ['Charter colony', ['Charter colony is one of three classes of colonial government established in the 17th century English colonies in North America, the other classes being proprietary colony and royal colony.', ' The colonies of Rhode Island, Connecticut, and Massachusetts Bay were charter colonies.', ' In a charter colony, Britain granted a charter to the colonial government establishing the rules under which the colony was to be governed.', ' The charters of Rhode Island and Connecticut granted the colonists significantly more political liberty than other colonies.', ' Rhode Island and Connecticut continued to use their colonial charters as their State constitutions after the American Revolution.']], ['Canada under British rule', ['Canada first came under British rule with the Treaty of Paris (1763) which ceded New France, of which Canada was a part, to the British Empire.', ' Gradually, other territories, colonies, and provinces that were part of British North America would be added to Canada.', ' The Royal Proclamation of 1763 enlarged the colony of Canada under the name of the Province of Quebec, which with the Constitutional Act 1791 became known as The Canadas.', ' With the Act of Union 1840 Upper and Lower Canada were joined to become the United Province of Canada.', ' Later, with Confederation in 1867, the British maritime colonies of New Brunswick and Nova Scotia were joined with the British colony of Canada to form the Dominion of Canada, which was subsequently divided into four provinces, Ontario, Quebec, New Brunswick, and Nova Scotia.', \" A number of other British colonies, such as Newfoundland and British Columbia, and large territories such as Rupert's Land initially remained outside of the newly formed federation.\", ' Over time, the remaining colonies and territories within British North America came under the control of Canada until the current geographic extent of the country was reached when Newfoundland and Labrador joined Canada in 1949.', ' Although confederation in 1867 led to an enlarged Dominion with increased autonomy over domestic affairs, Canada still remained a colony within the British Empire and was thus subordinate to the British Parliament until the enactment of the Statute of Westminster in 1931.', ' This statute recognized Canada as an independent peer coequal with the United Kingdom, and thus provided the Parliament of Canada with legislative sovereignty over all federal matters except the power to change the constitutional laws of Canada which remained under the purview of the Parliament of the United Kingdom.', \" Canada's final vestige of legal dependence on the United Kingdom was terminated in 1982 with the enactment of the Canada Act, subsequently providing Canada with full legal sovereignty completely independent of the United Kingdom.\"]], ['Stamp Act Congress', ['The Stamp Act Congress or First Congress of the American Colonies was a meeting held between October 7 and 25, 1765 in New York City, consisting of representatives from some of the British colonies in North America; it was the first gathering of elected representatives from several of the American colonies to devise a unified protest against new British taxation.', ' Parliament had passed the Stamp Act, which required the use of specially stamped paper for legal documents, playing cards, calendars, newspapers and dice for virtually all business in the colonies, and was going into effect on November 1.']], ['Province of New Jersey', ['The Province of New Jersey was one of the Middle Colonies of Colonial America and became the U.S. state of New Jersey in 1776.', ' The province had originally been settled by Europeans as part of New Netherland, but came under English rule after the surrender of Fort Amsterdam in 1664, becoming a proprietary colony.', ' The English then renamed the province after the Isle of Jersey in the English Channel.', ' The Dutch Republic reasserted control for a brief period in 1673–1674.', ' After that it consisted of two political divisions, East Jersey and West Jersey, until they were united as a royal colony in 1702.', ' The original boundaries of the province were slightly larger than the current state, extending into a part of the present state of New York, until the border was finalized in 1773.']], ['New York – New Jersey Line War', ['The New York – New Jersey Line War (also known as the N.J. Line War) refers to a series of skirmishes and raids that took place for over half a century between 1701 and 1765 at the disputed border between two American colonies, the Province of New York and the Province of New Jersey.']], ['Proprietary colony', ['A proprietary colony was a type of British colony mostly in North America and the Caribbean in the 17th century.', ' In the British Empire, all land belonged to the ruler, and it was his prerogative to divide.', ' Therefore, all colonial properties were partitioned by royal charter into one of four types: proprietary, royal, joint stock, or covenant.', ' King Charles II used the proprietary solution to reward allies and focus his own attention on Britain itself.', ' He offered his friends colonial charters which facilitated private investment and colonial self-government.', ' The charters made the proprietor the effective ruler, albeit one ultimately responsible to English law and the king.', ' Charles II gave New Netherland to his younger brother The Duke of York, who named it New York.', ' He gave an area to William Penn who named it Pennsylvania.']], ['Stamp Act 1765', ['The Stamp Act of 1765 (short title \"Duties in American Colonies Act 1765\"; 5 George III, c. 12) was an Act of the Parliament of Great Britain that imposed a direct tax on the colonies of British America and required that many printed materials in the colonies be produced on stamped paper produced in London, carrying an embossed revenue stamp.', ' Printed materials included legal documents, magazines, playing cards, newspapers, and many other types of paper used throughout the colonies.', ' Like previous taxes, the stamp tax had to be paid in valid British currency, not in colonial paper money.', \" The purpose of the tax was to help pay for troops stationed in North America after the British victory in the Seven Years' War and its North American theater of the French and Indian War.\", ' The Americans said that there was no military need for the soldiers because there were no foreign enemies on the continent, and the Americans had always protected themselves against Indians.', ' They suggested that it was actually a matter of British patronage to surplus British officers and career soldiers who should be paid by London.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.791\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab5ecd75542992aa134a3e6', 'answer': 'John de Mol Jr.', 'question': 'Which Dutch media tycoon transferred all of its media activities to Talpa Holding?', 'supporting_facts': [['Talpa Holding', 0], ['John de Mol Jr.', 0]], 'context': [['Talpa Radio', ['Talpa Radio (Formerly: \"538 Group\", pronounced \"Vijf-Drie-Acht-Groep\" in Dutch and \"Sky Radio Group\") is a radio company of Talpa Media, in which various radio and television activities are housed.', ' The group was founded on January 1, 2012, as a result of an acquisition of Radio 538 by Talpa from RTL Nederland.']], ['Stage Entertainment', ['Stage Entertainment is a live entertainment company founded by Dutch media tycoon and theatrical producer, Joop van den Ende.', ' It is based in Amsterdam, Netherlands.', ' Stage Entertainment is in business with offices and theatres in the Netherlands, Germany, Spain, the United Kingdom, the United States, Russia, France and Italy.', ' The group produces musical shows in large theatres, comprising licensed productions from international partners as well as original, in-company storyline.']], ['Talpa Holding', ['Talpa Holding is the company in which John de Mol Jr. has transferred all of its media activities.', ' Besides John de Mol, who as majority shareholder owns 80% of the company, Rabo Participaties owns a 20% stake in the media company.', ' The holding company has amongst the 538 group with radio station Radio 538.', ' Talpa Media, formerly part of Talpa Holding, has been sold to ITV plc and is a separate business unit within ITV Studios.']], ['Sky Radio', ['Sky Radio is a Dutch commercial radio station playing non-stop Adult Contemporary-pop music and is owned by a joint venture between Talpa Holding and Telegraaf Media Groep.', ' The station slogan is \"Your favourite playlist!\"', '.', ' The station primarily plays pop and rock music from the 1980s through to the present.']], ['John de Mol Jr.', ['Johannes Hendrikus Hubert \"John\" de Mol Jr. (born 24 April 1955 in The Hague) is a Dutch media tycoon.', ' De Mol is one of the men behind production companies Endemol and Talpa.']], ['Tien (TV channel)', ['Tien (meaning \"Ten\" in Dutch), previously known as Talpa, was the name of a commercial television channel in the Netherlands.', ' Tien opened on August 13, 2005 as Talpa, following a name dispute with SBS Broadcasting.', ' SBS owned the trademark \"TV10\" and objected to the use of the word Tien.', ' The owner of Tien, Dutch media mogul John de Mol, decided to rebrand the channel as \"Talpa\", the Latin word for \"mole\", which is \"mol\" in Dutch.', \" Subsequently Talpa became the name of De Mol's holding company.\"]], ['Radio 538', ['Radio 538 (Dutch: \"vijf-drie-acht\" ) is a Dutch commercial radio station that has evolved since 1992.', ' \"538,\" refers to the wavelength that Radio Veronica was broadcast on in the seventies.', ' This station was intended for the younger generation and is owned by Talpa Holding and Telegraaf Media Groep.', ' It has a broad variety of genres including the Top 40, dance, R&B, pop, rock and recent hits.', ' Traditionally, the station was only obtained through cable, however in 1998 the station upgraded to a different frequency package, allowing for different frequencies by region.', ' Radio 538 started the hip-hop and contemporary R&B program \"Juize\", which developed into the radio station Juize.FM on 18 July 2004.', ' Later, in 2011, Radio 538 created a sister station called TV 538.']], ['TV 538', ['TV 538 is a music television channel that airs music videos and live coverage of its radio broadcasts.', ' Dutch radio station 538 launched the TV channel on 4 July 2011.', ' The channel is owned by a joint venture between Talpa Holding and Telegraaf Media Groep .', ' It broadcasts 24 hours a day and airs across the Netherlands.']], ['Radio Veronica (Talpa Radio)', ['Radio Veronica is a Dutch commercial radio station of a joint venture between Talpa Holding and Telegraaf Media Groep .', ' The station runs mainly music from the 1980s, 1990s and 2000s.', ' The station can be compared with Absolute Radio in the United Kingdom.']], ['Villa Isola', ['Villa Isola (now Bumi Siliwangi) is an art-deco building in the northern part of Bandung, the capital of West Java province of Indonesia.', ' Overlooking the valley with the view of the city, Villa Isola was completed in 1933 by the Dutch architect Wolff Schoemaker for the Dutch media tycoon Dominique Willem Berretty, the founder of the Aneta press-agency in the Dutch East Indies.', \" The original purpose of the building was for Berretty's private house, but then it was transformed into a hotel after his death and now it serves as the headmastership office of the University of Education Indonesia.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.792\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8789115542994846c1cd9a', 'answer': 'Pollywood', 'question': 'Imran Khan has worked in what type of films refering to the Pashto Language film industry?', 'supporting_facts': [['Imran Khan (Pakistani actor)', 0], ['Pashto cinema', 0]], 'context': [['Khan Abdul Ghani Khan', ['Ghani Khan (Pashto: غني خان) \\u200e (1914–1996) was a Pakistani Pashto language poet, artist, writer, politician and Philosopher of the 20th century.', ' He was a son of Khan Abdul Ghaffar Khan and older brother of Khan Abdul Wali Khan.']], ['Imran Khan (Pakistani actor)', ['Imran Khan (better known as just Imran) is a Pakistani film actor who has worked in Lollywood and Pollywood films.']], ['Imran Khan (Indian actor)', ['Imran Khan (] ; born Imran Pal 13 January 1983) is an American-born film actor, who appears in Hindi films.', ' He is the nephew of actor Aamir Khan and director-producer Mansoor Khan, and the grandson of director-producer Nasir Hussain.', ' He appeared as a child artist in the films \"Qayamat Se Qayamat Tak\" (1988) and \"Jo Jeeta Wohi Sikander\" (1992).']], ['Laaj', ['Laaj (Urdu: \\u200e ) is a 2003 Pakistani Urdu language film which was directed by Rauf Khalid.', ' The film starred Zara Sheikh and Imran Khan in its lead roles.', \" Film's music is composed by Amjad Bobby.\"]], ['Mohammad Imran Pratapgarhi', ['Mohammad Imran Pratapgarhi Urdu: محمّد عمران خان\\u200e Hindi: इमरान प्रतापगढ़ी originally known as Mohammad Imran Khan is a famed Urdu language and Hindi language Poet who has gained prominence among the audience through his revolutionary poems.', ' The three times National Award Winner for debate and poetry, he has a firm belief in following his heart.', ' His work has a dominance in framing verses for sharp socio-political distortions, country- love, brotherhood and religious - social harmony fragrance broke.']], ['Cinema of Bangladesh', ['The cinema of Bangladesh is the Bengali language film industry based in Dhaka, Bangladesh.', ' It has often been a significant film industry since the early 1970s and is frequently referred to as \"Dhallywood\" (Bengali: ঢালিউড ), which is a portmanteau of the words Dhaka and Hollywood.', ' The dominant style of Bangladeshi cinema is melodramatic cinema, which developed from 1947 to 1990 and characterizes most films to this day.', ' Cinema was introduced in Bangladesh in 1898 by Bradford Bioscope Company, credited to have arranged the first film release in Bangladesh.', ' Between 1913 and 1914, the first production company named Picture House was opened.', ' A short silent film titled \"Sukumari\" (\"The Good Girl\") was the first produced film in the region during 1928.', ' The first full-length film \"The Last Kiss\", was released in 1931.', ' From the separation of Bangladesh from Pakistan, Dhaka is the center of Bangladeshi film industry, and generated the majority share of revenue, production and audiences. \"', 'The Face and the Mask\", the first Bengali language Bangladeshi full-length feature film was produced in 1956.', ' The 1960s, 1970s, 1980s and the first half of the 1990s were the golden years for Bangladeshi films as the industry produced many successful films.', ' But during then many of the films were unofficial remake of Indian films.']], ['Jaane Tu... Ya Jaane Na', ['Jaane Tu... Ya Jaane Na (translation: \"Whether you know... or not\") is a 2008 Indian coming of age romantic drama film, written and directed by Abbas Tyrewala.', \" The film stars Imran Khan and Genelia D'Souza in pivotal roles.\", \" Produced by Mansoor Khan, Aamir Khan, it marks the directional debut of Abbas Tyrewala, the debut of Imran Khan (Aamir Khan's nephew) and Prateik Babbar as actors, and the re-appearance of D'Souza in Hindi cinema.\", ' Released on 4 July 2008, the film received positive reviews, and was successful at the box office.', ' The music is by A. R. Rahman.']], ['57th Filmfare Awards', ['The 57th Filmfare Awards were held on January 29, 2012 at Film City, Mumbai honoring the best film of 2011 from the Hindi-language film industry (commonly known as Bollywood).', ' The ceremony was jointly hosted by Shahrukh Khan and Ranbir Kapoor.', ' Incidentally, both of them have hosted the award ceremonies previously but with different co-hosts (Khan with Saif Ali Khan, Kapoor with Imran Khan), hence making it the first time for this pair to host the show.']], ['Cinema of Pakistan', ['The Cinema of Pakistan or Pakistani cinema (Urdu: \\u200e ) refers to the filmmaking industry in Pakistan.', ' Pakistan is home to several film studios centres, primarily located in its two largest cities - Karachi and Lahore.', ' Pakistani cinema has played an important part in Pakistani culture, and in recent years has begun flourishing again after years of decline, delivering entertainment to audiences in Pakistan and expatriates abroad.', ' Several film industries are based in Pakistan, which tend to be regional and niche in nature.', ' Over 10,000 Urdu feature-films have been produced in Pakistan since 1948, as well as over 8000 Punjabi, 6000 Pashto and 2000 Sindhi feature-length films.', ' The first film ever produced was \"Husn Ka Daku\" in 1930, directed by Abdur Rashid Kardar in Lahore.', ' The first Pakistani-film produced was \"Teri Yaad\", directed by Daud Chand in 1948.', \" Between 1947 and 2007, Pakistani cinema was based in Lahore, home to the nation's largest film industry (nicknamed Lollywood).\", ' Pakistani films during this period attracted large audiences and had a strong cult following, was part of the cultural mainstream, widely available and imitated by the masses.', \" During the early 1970s, Pakistan was the world's fourth largest producer of feature films.\", ' However, between 1977 and 2007, the film industry of Pakistan went into decline due to Islamization, strengthening of censorship laws and an overall lack of quality.', ' Throughout the 1980s and 1990s, the film industry went through several periods of ups and downs, a reflection of its dependency on state funding and incentives.', ' By 2000, the film industry in Lahore had collapsed and saw a gradual shift of Pakistani actors, actresses, producers and filmmakers from Lahore to Karachi.', \" By 2007, the wounds of Pakistan's collapsed film industry began to heal and Karachi had cemented itself as the centre of Pakistani cinema.\", ' Quality and new technology led to an explosion of alternative form of Pakistani cinema.', ' The shift has been seen by many as the leading cause for the \"resurgence of Pakistani cinema\".', ' Despite the industry crisis starting in the mid-1980s, Pakistani films have retained much of its distinctive identity.', ' Since the shift to Karachi, Pakistani films have once again began attracting a strong cult following.']], ['Pashto cinema', ['Pashto cinema (Urdu: \\u200e , Pashto: د پښتو سينما\\u200e ), also known by its sobriquet Pollywood (Pashto: پالېوډ\\u200e ), refers to the Pashto language film industry of Pakistani cinema based in Peshawar, Khyber Pakhtunkhwa, Pakistan.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.793\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a8a1dbc55429970aeb7025a', 'answer': 'Sulla', 'question': 'Lex Antonia nullified the dictatorial laws set up by which Roman general?', 'supporting_facts': [['Lex Antonia', 2], ['Lex Antonia', 3], ['Sulla', 0]], 'context': [['Sulla', ['Lucius Cornelius Sulla Felix ( ; c. 138 BC – 78 BC), known commonly as Sulla, was a Roman general and statesman.', ' He had the distinction of holding the office of consul twice, as well as reviving the dictatorship.', ' Sulla was a skillful general, achieving numerous successes in wars against different opponents, both foreign and Roman.', ' He was awarded a grass crown, the most prestigious Roman military honor, during the Social War.']], ['Lex specialis', ['Lex specialis, in legal theory and practice, is a doctrine relating to the interpretation of laws and can apply in both domestic and international law contexts.', ' The doctrine states that if two laws govern the same factual situation, a law governing a specific subject matter (\"lex specialis\") overrides a law governing only general matters (\"lex generalis\").', ' The situation ordinarily arises with regard to the construction of earlier-enacted specific legislation when more general legislation is later passed.', ' However, then, the doctrine called \"lex posterior derogat legi priori\" may also apply, the younger law overriding the older law.']], ['Iullus Antonius', ['Iullus Antonius (45 BC – 2 BC), also known as Iulus, Julus or Jullus, was a personage in Ancient Rome.', \" He was the second son of Roman general Mark Antony and Antony's third wife Fulvia.\", ' He is best known for being the famous lover of Julia the Elder.', \" He was the full brother of Marcus Antonius Antyllus, half-brother of Clodia Pulchra (the first wife of Augustus) through his mother's first marriage, half-brother of Antonia Major and Antonia Minor through his father's marriage to Octavia Minor, and half-brother of Alexander Helios, Cleopatra Selene\\xa0II and Ptolemy Philadelphus through his father's marriage to Cleopatra\\xa0VII.\", ' His stepsiblings were Marcellus, Claudia Marcella Major (later his wife), Caesarion and Claudia Marcella Minor.', ' He was also stepson to Octavia Minor (sister of Augustus) and Cleopatra\\xa0VII.']], ['Lex Junia Norbana', ['In Roman Law, Lex Iunia Norbana of 19 AD classified all freedmen into two classes according to their mode of enfranchisement: enfranchised citizens, (freedmen who enjoyed Roman citizenship) and enfranchised Latini (freedmen who had only Latin rights).', ' Braund, D., Augustus to Nero (Routledge Revivals): A Sourcebook on Roman History, 31 BC-AD 68 (2015), [710] Freedmen would be granted only Latin rights if the manumission of the slave failed to meet any of the conditions set out by the lex Aelia Sextia of 4 AD for it to confer Roman citizenship.', ' This provided that for the freedman to acquire Roman citizenship a slave had to be manumitted at the age of 30 or older, the owner had to have quiritary ownership and the ceremony had to be public.', ' For slaves under the age of thirty, the manumission had to be approved by a special council.', ' The manumission of slaved who had been enslaved because of crimes would raise them only to the position of dedititii (war captives).', ' ^Thus, the Lex Iunia Norbana made the slaves who were not eligible for Roman citizens as per the lex Aelia Sextia enfranchised Latins.', ' The law retained the dedititii.', ' A clause of the law \"took away from these Latini Juniani, as they were called, the capacity of making a testament, taking under a testament, and being appointed tutores by a testament.\"']], ['Joe Laws', ['Joe Roy Laws (June 16, 1911 – August 22, 1979) was an American football player.', ' He played his entire career with the Green Bay Packers, winning three World Championships, and was inducted into the Green Bay Packers Hall of Fame in 1972.', ' Prior to joining the Packers, Laws attended the University of Iowa where he was a member of Sigma Pi fraternity.', ' While at Iowa he was named All-Big Ten quarterback and the Big Ten Most Valuable Player in 1933.', \" On December 17, 1944 Joe Laws set an NFL postseason record (since broken), by intercepting 3 passes in the Packers' 14-7 victory over the Giants in the league title game.\"]], ['Lex Irnitana', ['The lex Irnitana is a collection of six bronze tablets containing fragments of Roman municipal laws found in 1981 near El Saucejo, Spain.', ' Together with the \"Lex Salpensana\" and the \"Lex Malacitana\" they provide the most complete version of the \"lex Flavia municipalis\", Flavian municipal law.', ' and have allowed new insights into the workings of Roman law.', ' The tablets are exhibited in the Archeological Museum of Seville.', ' Since the tablets provide the only surviving copy of large parts of the Flavian municipal law, it has provided new insights into the procedural side of municipal courts.']], ['Lex Antonia de Termessibus', ['The Lex Antonia de Termessibus was a Roman law passed in 71 or 68 BC, at the initiative of the tribune Gaius Antonius.']], ['Aghbugha I Jaqeli', ['Aghbugha I Jaqeli (Georgian: აღბუღა I ჯაყელი ) (died 1395) was a Georgian prince (\"mtavari\") and Atabeg of Samtskhe from 1389 to 1395.', ' Aghbugha was a Son of Prince Shalva.', \" After his father's death Aghbugha was appointed as co-ruler (he ruled with his uncle Beka I) of Meskheti by Georgian king Bagrat V.\", ' During 1381-1386 he renewed The book of laws which was established by his Great-great-grandfather, Beka Jaqeli.', ' This book firstly was called \"Aghbugha\\'s law\", then \"Book of laws set by Beka-Aghbugha\".']], ['Lex Antonia', ['Lex Antonia (Latin for \"Antonine law\", sometimes presented plurally as the leges Antoniae, \"Antonine laws\") was a law established in ancient Rome in April 44 BC.', ' It was proposed by Mark Antony and passed by the Roman Senate, following the assassination of Julius Caesar.', ' It formally abolished the Dictatorship.', ' It was the second law to do so (the first being passed after the Second Punic War, replacing the Dictatorship with the final decree of the Senate); however, the earlier law had essentially been nullified by the subsequent Dictatorships of Sulla and Caesar.']], ['Lex Burgundionum', ['The Lex Burgundionum (Latin for Burgundian Laws, also \"Lex Gundobada\") refers to the law code of the Burgundians, probably issued by king Gundobad.', ' It is influenced by Roman law and deals with domestic laws concerning marriage and inheritance as well as regulating weregild and other penalties.', ' Interaction between Burgundians is treated separately from interaction between Burgundians and Gallo-Romans.', \" The oldest of the 14 surviving manuscripts of the text dates to the 9th century, but the code's institution is ascribed to king Gundobad (died 516), with a possible revision by his successor Sigismund (died 523).\", ' The \"Lex Romana Burgundionum\" is a separate code, containing various laws taken from Roman sources, probably intended to apply to the Burgundians\\' Gallo-Roman subjects.', ' The oldest copy of this text dates to the 7th century.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.794\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a77e70f5542992a6e59dfeb', 'answer': 'The Tempest', 'question': 'What is the title of the 1979 film adaptation of William Shakespeare\\'s play in which the English poet, actor, political activist and dramatist who wrote wrote a number of book-length polemical poems such as \"Autogeddon\", \"Falling for a Dolphin\" and \"Whale Nation\" played a main character?', 'supporting_facts': [['The Tempest (1979 film)', 0], ['The Tempest (1979 film)', 1], ['Heathcote Williams', 0], ['Heathcote Williams', 1]], 'context': [['The Tempest (1979 film)', [\"The Tempest is a 1979 film adaptation of William Shakespeare's play of the same name.\", ' Directed by Derek Jarman, with Heathcote Williams as Prospero, it also stars Toyah Willcox, Jack Birkett and Helen Wellington-Lloyd from Jarman\\'s previous feature, \"Jubilee\" (1977), as well as his long-time cohort Karl Johnson.']], ['William Shakespeare', [\"William Shakespeare ( ; 26 April 1564 (baptised)\\xa0– 23 April 1616) was an English poet, playwright, and actor, widely regarded as the greatest writer in the English language and the world's pre-eminent dramatist.\", ' He is often called England\\'s national poet, and the \"Bard of Avon\".', ' His extant works, including collaborations, consist of approximately 38 plays, 154 sonnets, two long narrative poems, and a few other verses, some of uncertain authorship.', ' His plays have been translated into every major living language and are performed more often than those of any other playwright.']], ['Henry Carey (writer)', ['Henry Carey (c. 26 August 1687 – 5 October 1743) was an English poet, dramatist and song-writer.', ' He is remembered as an anti-Walpolean satirist and also as a patriot.', ' Several of his melodies continue to be sung today, and he was widely praised in the generation after his death.', ' Because he worked in anonymity, selling his own compositions to others to pass off as their own, contemporary scholarship can only be certain of some of his poetry, and a great deal of the music he composed was written for theatrical incidental music.', ' However, under his own name and hand, he was a prolific song writer and balladeer, and he wrote the lyrics for almost all of these songs.', ' Further, he wrote numerous operas and plays.', ' His life is illustrative of the professional author in the early 18th century.', ' Without inheritance or title or governmental position, he wrote for all of the remunerative venues, and yet he also kept his own political point of view and was able to score significant points against the ministry of the day.', ' Further, he was one of the leading lights of the new \"Patriotic\" movement in drama.']], ['Holy Sonnets', ['The Holy Sonnets—also known as the Divine Meditations or Divine Sonnets—are a series of nineteen poems by the English poet John Donne (1572–1631).', \" The sonnets were first published in 1633—two years after Donne's death.\", ' The poems are sonnets and are predominantly in the style and form prescribed by Renaissance Italian poet Petrarch (or Francesco Petrarca) (1304–1374) in which the sonnet consisted of two quatrains (four-line stanzas) and a sestet (a six-line stanza).', ' However, several rhythmic and structural patterns as well as the inclusion of couplets are elements influenced by the sonnet form developed by English poet and playwright William Shakespeare (1564–1616).']], ['Samuel Taylor Coleridge', ['Samuel Taylor Coleridge ( ; 21 October 177225 July 1834) was an English poet, literary critic, philosopher and theologian who, with his friend William Wordsworth, was a founder of the Romantic Movement in England and a member of the Lake Poets.', ' He wrote the poems \"The Rime of the Ancient Mariner\" and \"Kubla Khan\", as well as the major prose work \"Biographia Literaria\".', ' His critical work, especially on William Shakespeare, was highly influential, and he helped introduce German idealist philosophy to English-speaking culture.', ' Coleridge coined many familiar words and phrases, including suspension of disbelief.', ' He was a major influence on Ralph Waldo Emerson and American transcendentalism.']], ['Isabella Cervoni', [\"Isabella Cervoni (Colle Val d'Elsa, 1575–1600) was an Italian poet of the Counter-Reformation period, active between 1590 and 1600.\", \" She wrote encomiastic and polemical poems addressed to numerous secular and religious dignitaries of the Italian Renaissance, including Pope Clement VIII, Maria de' Medici, Christina of Lorraine and Henry IV of France.\", ' She was praised for her talent and ambition by Cristoforo Bronzini in his 1625 dialogue \"Della dignità delle donne, dialogo…settimana prima e giornata quarta\" as having \"given the world many beautiful and spiritual compositions\" despite her \"most tender age.\"']], ['Shakespeare bibliography', ['William Shakespeare (1564–1616) was an English poet and playwright.', ' He wrote approximately 38 plays and 154 sonnets, as well as a variety of other poems.']], ['Charles Goodall (poet)', ['Charles Goodall (1671—May 11, 1689) is a minor English poet.', ' A student of Eton College and then Merton College, Oxford, he wrote a number of romantic and erotic poems referring to male students at said colleges.', ' In 1689, the year of his death, he put together a collection entitled \"Poems and Translations\" which contains 33 poems with male-male subject matter, eleven regarding women, and 13 to a mistress named \\'Idera\\' (considered probably imaginary).', ' A number of the homoerotic poems have been rewritten to remove the same-sex subject matter.']], ['Sonnet 154', ['As the last in the famed collection of sonnets written by English poet and playwright William Shakespeare from 1592 to 1598, Sonnet 154 is most often thought of in a pair with the previous sonnet, number 153.', ' As A. L. Rowse states in \"Shakespeare\\'s Sonnets: The Problems Solved\", Sonnets 153 and 154 \"are not unsuitably placed as a kind of coda to the Dark Lady Sonnets, to which they relate.\"', ' Rowse calls attention to the fact that Sonnets 153 and 154 \"serve quite well to round off the affair Shakespeare had with Emilia, the woman characterized as the Dark Lady, and the section of the Dark Lady sonnets\".', ' Shakespeare used Greek mythology to address love and despair in relationships.', ' The material in Sonnets 153 and 154 has been shown to relate to the six-line epigram by the Byzantine poet known as Marianus Scholasticus, who published a collection of 3,500 poems called \"The Greek Anthology\".', \" When translated, the epigram resembles Sonnets 153 and 154, addressing love and the story of Cupid, the torch, and the Nymph's attempt to extinguish the torch.\"]], ['Heathcote Williams', ['John Henley Heathcote-Williams (15 November 1941 – 1 July 2017), known as Heathcote Williams, was an English poet, actor, political activist and dramatist.', ' He wrote a number of book-length polemical poems including \"Autogeddon\", \"Falling for a Dolphin\" and \"Whale Nation\", which in 1988 became, according to Philip Hoare, \"the most powerful argument for the newly instigated worldwide ban on whaling.\"', ' Williams invented his idiosyncratic \"documentary/investigative poetry\" style which he put to good purpose bringing a diverse range of environmental and political matters to public attention.', ' His last published work, \"American Porn\" was a critique of the American political establishment and the election of President Donald Trump: Publication date was the date of Trump\\'s inauguration (20 January 2017).', ' In June 2015, he published a book-length investigative poem about the \"Muslim Gandhi\", Khan Abdul Ghaffar Khan, \"Badshah Khan\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.794\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7b1c6b55429931da12c9ca', 'answer': 'John Douglas \"Johnny\" Edwards', 'question': 'Johnny Edwards and Ian Anderson are singers, who had joined the greater number of bands?', 'supporting_facts': [['Johnny Edwards (musician)', 0], ['Ian Anderson', 0]], 'context': [['Belize National Youth Chess Foundation', ['The Belize National Youth Chess Foundation (B.N.Y.C.F.) was co-founded by Ian & Ella Anderson in the summer of 2007 as a not-for-profit organization and with a small army of volunteers it spread throughout the country.', ' The game of Chess has been around for a very long time but in Belize there were no formal organizations and no figures to indicate how many people were playing the game.', ' Building on the founding by Mr. Robert Landolfi and Mr. Glen Reneau of the first school chess club at Hummingird Elementary and the Belize Association of Chess Players in Belize City, due to the efforts of the B.N.Y.C.F. there are now teams ranging from the most southern villages in Toledo District to the most northern villages along the Belize-Mexico border in the Corozal District.', ' Since 2007, the organization has more than 50 active chess clubs and over 1400 players around the country and it functions all year round.', ' An interview with Ian Anderson, Co-Chair, reveals that chess is not only a pastime or hobby in Belize; it can and should be used “as an educational tool to help develop the minds of primary school students.”', ' The B.N.Y.C.F. has worked with primary schools to successfully integrate chess as a part of the curriculum of the primary schools in Belize.', ' Within one year the game of Chess became the fastest growing sport in the country.', ' As part of its efforts to promote this sport, the B.N.Y.C.F. assisted the Belize Chess Federation to become active again in 2008 by updating fees due to FIDE, the World Chess affiliate.']], ['Dead to Me', ['Dead to Me is a punk rock band from San Francisco, founded by vocalist/guitarist Jack Dalrymple, drummer Brandon Pollack (both from the band One Man Army) and bassist/vocalist Chicken of Western Addiction.', \" Early on, Pollack was replaced by Chicken's cousin Ian Anderson on drums and Nathan Grice joined as a second guitarist.\"]], ['Mike Vickers', ['Michael \"Mike\" Vickers (born 18 April 1940) is a British musician who came to prominence as guitarist, flautist and saxophonist with the 1960s band, Manfred Mann.', ' He was born in Southampton, Hampshire, England.', ' He originally played flute and saxophone but with the increasing popularity of guitars in bands it was decided that Manfred Mann should have a guitarist in its line-up.', ' Vickers volunteered for this role but he was always happiest playing woodwind.', ' His tough flute soloing on hard blues tracks such as \"Without You\" prefigured the work of Ian Anderson with Jethro Tull five years later.', ' As the group were all multi-instrumentalists who delighted in instrumental solos, multi-tracking was used to allow Vickers to perform on guitar and woodwind on the same recordings, while drummer Mike Hugg similarly doubled on vibraphone.']], ['King Kobra III', ['King Kobra III, released in 1988 on New Renaissance Records, was the first and last album by the Edwards, Michael-Phillips, Northrup, Hart and Appice line-up of King Kobra.', ' After the demise of the original line-up, remaining members Carmine Appice and David Michael-Phillips teamed up with Johnny Edwards, Jeff Northrup and Larry Hart, all 3 members of the Sacramento, CA band Northrup at the time.']], ['Gerald Bostock', ['Gerald Bostock is a fictional character originally created by Ian Anderson for his band Jethro Tull\\'s 1972 concept album, \"Thick as a Brick\"; Bostock is credited with writing the lyrics to the album (though Anderson in fact authored them himself).', ' Bostock is also the focus of Anderson\\'s 2012 solo album, \"Thick as a Brick 2: Whatever Happened to Gerald Bostock?', '\", as well as the purported lyricist for Anderson\\'s 2014 solo album \"Homo Erraticus\".']], ['Wild Horses (US rock band)', ['Wild Horses was a band that originally featured former Buster Brown and Montrose members Johnny Edwards and James Kottak.', \" The band also featured James Kottak's former Kingdom Come bandmate Rick Steier.\", ' The band went through at least two bassists: Chris Lester and Jeff Pilson.']], ['Afternoon Records', ['Afternoon Records is a record label based in Minneapolis, Minnesota.', \" The label was founded by Ian Anderson and Michael M. Sandstedt in 2003, the year of Ian's graduation from high school.\", ' Ian wanted to create a platform for his high school band \"Aneuretical\", and others.']], ['Johnny Edwards (musician)', ['John Douglas \"Johnny\" Edwards is an American rock singer who sang for the bands Buster Brown, Montrose, King Kobra, Wild Horses, Northrup, Royal Jelly and is best known as the second lead singer of the rock band Foreigner.']], ['Unusual Heat', ['Unusual Heat is the seventh studio album by British-American rock band Foreigner, released on 14 June 1991 by Atlantic Records.', ' Recorded at several different studios across the state of New York and England, and produced by Terry Thomas and Mick Jones, it was the only album with lead singer Johnny Edwards.', ' He replaced original lead singer Lou Gramm after the latter had parted company in 1990.', ' \"Unusual Heat\" was the last album to feature bass guitarist Rick Wills, who joined the band in 1979, and drummer Dennis Elliott, who was a founding member.']], ['Ian Anderson', ['Ian Scott Anderson, MBE (born 10 August 1947) is a Scottish-born musician, singer, songwriter and multi-instrumentalist best known for his work as the lead vocalist, flautist and acoustic guitarist of British rock band Jethro Tull.', ' Anderson plays several other musical instruments, including keyboards, bass guitar, bouzouki, balalaika, saxophone, harmonica, and a variety of whistles. His solo work began with the 1983 album \"Walk into Light\", and since then he released another five works, including the sequel to the Jethro Tull album \"Thick as a Brick\" (1972) in 2012, entitled \"Thick as a Brick 2\".']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.796\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7337715542991f9a20c690', 'answer': \"Coal Miner's Daughter\", 'question': 'For which Thomas Rickman film did Sissy Spacek win an Academy Award for Best Actress?', 'supporting_facts': [['Thomas Rickman (writer)', 0], [\"Coal Miner's Daughter (film)\", 1]], 'context': [['Killings (short story)', ['Killings is a short tale written by Andre Dubus in 1979.', ' The short story entails how a man seeks revenge after the death of his son in cold blood.', ' In 2001, the story was adapted into Todd Field\\'s film, \"In the Bedroom\".', ' The film starred Sissy Spacek, Tom Wilkinson, and Marisa Tomei, and was nominated for five Academy Awards – Best Picture, Actor in a Leading Role (Wilkinson), Actress in a Leading Role (Spacek), Actress in a Supporting Role (Tomei), and Best Writing, Screenplay Based on Material Previously Published (Robert Festinger & Field).', ' After the film\\'s release the story was republished in a collection called \"In the Bedroom\" for which Field wrote the preface.']], [\"Coal Miner's Daughter (film)\", [\"Coal Miner's Daughter is a 1980 American biographical film which tells the story of country music singer Loretta Lynn.\", ' It stars Sissy Spacek as Loretta, a role that earned her the Academy Award for Best Actress.', \" Tommy Lee Jones as Loretta's husband Mooney Lynn, Beverly D'Angelo and Levon Helm also star.\", ' The film was directed by Michael Apted.']], ['Sissy Spacek', ['Mary Elizabeth \"Sissy\" Spacek ( ; born December 25, 1949) is an American actress and singer.', ' She began her career in the early 1970s and first gained attention for her role in the film \"Badlands\" (1973).', ' Her major breakthrough came in 1976 when she played the title character of Carrie White in Brian De Palma\\'s horror film \"Carrie\", based on the first novel by Stephen King, for which she earned an Oscar nomination (a rare feat for an actor or actress in a horror movie).', ' She won the Academy Award for Best Actress for her portrayal of Loretta Lynn in the 1980 film \"Coal Miner\\'s Daughter,\" and also earned a Grammy nomination for the song \"Coal Miner\\'s Daughter\" from the film\\'s soundtrack.', ' She went on to receive further Oscar nominations for her roles in \"Missing\" (1982), \"The River\" (1984) and \"Crimes of the Heart\" (1986).', ' \"Coal Miner\\'s Daughter\" and \"Crimes of the Heart\" also won her the Golden Globe Award for Best Actress in a Musical or Comedy.']], ['Cate Blanchett', ['Catherine Elise Blanchett, {\\'1\\': \", \\'2\\': \", \\'3\\': \", \\'4\\': \"} ( ; born 14 May 1969) is an Australian actress and theatre director.', ' She has received international acclaim and many accolades, including two Academy Awards, three Golden Globe Awards, three BAFTA Awards, six AACTA Awards, and three Screen Actors Guild Awards. Blanchett came to international attention for her role as Elizabeth I of England in Shekhar Kapur\\'s 1998 film \"Elizabeth\", for which she won the BAFTA Award for Best Actress, the Golden Globe Award, and earned her first Academy Award for Best Actress nomination.', ' Her portrayal of Katharine Hepburn in Martin Scorsese\\'s 2004 film \"The Aviator\" brought her critical acclaim and many accolades, including the Academy Award for Best Actress in a Supporting Role, making her the only actor to win an Oscar for portraying another Oscar-winning actor.', ' In 2013, she starred as Jasmine Francis in Woody Allen\\'s \"Blue Jasmine\", for which she won numerous accolades including the Academy Award for Best Actress.']], ['Hot Rod (film)', ['Hot Rod is a 2007 American comedy film co-written, directed by, and starring members of The Lonely Island (Andy Samberg, Jorma Taccone and Akiva Schaffer).', ' The film stars Samberg as an amateur stuntman whose abusive step-father, Frank (Ian McShane) continuously mocks and disrespects him.', ' When Frank grows ill, Rod raises money for his heart operation by executing his largest stunt yet.', \" In addition to raising money for the operation, he also does so to win Frank's respect, by kicking his butt.\", ' The film also stars Taccone, Sissy Spacek, Will Arnett, Danny McBride, Isla Fisher and Bill Hader.', ' It was directed by Schaffer (in his directorial debut) and distributed by Paramount Pictures.']], ['The Grass Harp (film)', ['The Grass Harp is a 1995 American comedy-drama film based on the novella by Truman Capote; the screenplay was the final work of Oscar-winning screenwriter Stirling Silliphant.', \" The film was directed by Charles Matthau, and starred Piper Laurie, Sissy Spacek, the director's father Walter Matthau, Edward Furlong, and Nell Carter.\", ' Piper Laurie won the Best Supporting Actress award from the Southeastern Film Critics Association for her work on the film.']], ['JFK (soundtrack)', ['JFK is the original soundtrack of the 1991 Academy Award and Golden Globe Award-winning film, \"JFK\", starring Kevin Costner, Tommy Lee Jones, Kevin Bacon, Joe Pesci and Sissy Spacek.', ' The original score was composed by John Williams.']], ['Thomas Rickman (writer)', ['Thomas Rickman (sometimes credited as Tom Rickman) is an American film director and screenwriter known for such films as \"Coal Miner\\'s Daughter\", \"Hooper\", \"Tuesdays with Morrie\" and \"Truman\".']], ['Holly Hunter', ['Holly Hunter (born March 20, 1958) is an American actress and producer.', ' For her performance as Ada McGrath in the 1993 film \"The Piano\", she won the Academy Award for Best Actress, BAFTA Award for Best Actress in a Leading Role, AACTA Award for Best Actress in a Leading Role, Golden Globe Award for Best Actress – Motion Picture Drama, and the Cannes Best Actress Award.', ' She was also nominated for the Academy Award for Best Actress for \"Broadcast News\" (1987), and the Academy Award for Best Supporting Actress for \"The Firm\" (1993) and \"Thirteen\" (2003).']], ['Joyce Horman', ['Joyce Marie Horman (born December 3, 1944) is an American human rights activist.', ' She is known as the wife of journalist Charles Horman, who went missing in 1973 while the couple was living in Santiago, Chile.', ' Her search for what happened to him was chronicled in the 1982 film \"Missing\", in which she was portrayed by Sissy Spacek.', ' Spacek was nominated for an Academy Award for her performance as Horman.', ' Her family\\'s story was first told in the 1978 book by Thomas Hauser titled \"The Execution of Charles Horman: An American Sacrifice\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.796\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7d1d825542995ed0d165f5', 'answer': 'Lorman', 'question': 'In what city did Charlie Spiller play college football?', 'supporting_facts': [['Charlie Spiller', 2], ['Alcorn State University', 0]], 'context': [['1891 Purdue football team', ['The 1891 Purdue football team was an American football team that represented Purdue University during the 1891 college football season.', \" The team compiled a 4–0 record in the university's fourth season fielding an intercollegiate football team.\", ' For the 1891 season, Purdue hired Knowlton Ames as its football coach.', ' Ames played for Princeton from 1886 to 1889 and was considered one of the greatest players ever to play college football, after scoring 730 points for Princeton.', ' The 1891 Purdue team shut out all four opponents, outscoring Wabash, DePauw, Indiana, and Butler by a combined score of 192 to 0.', \" Purdue's 60–0 victory over Indiana was the first installment in a rivalry which later became noted for the award of the Old Oaken Bucket trophy.\"]], ['Ken McAlister', ['Kenneth H. McAlister (born April 15, 1960) is a former American football linebacker who played five seasons in the National Football League with the Seattle Seahawks, San Francisco 49ers and Kansas City Chiefs.', ' He played college basketball at the University of San Francisco and attended Oakland High School in Oakland, California.', ' He did not play college football and made the Seahawks roster in 1982.']], ['Jamal Anderson', ['Jamal Sharif Anderson (born September 30, 1972) is a former American football running back of the National Football League.', ' He was drafted by the Atlanta Falcons in the seventh round of the 1994 NFL Draft.', ' He played high school football at El Camino Real High School, where he was named to the CIF Los Angeles City Section 4-A All-City first team in 1989.', ' He went on to play college football at Moorpark College for the Moorpark College Raiders before playing at Utah.']], ['Seantrel Henderson', ['Seantrel Henderson (born January 21, 1992) is an American football offensive tackle for the Buffalo Bills of the National Football League (NFL).', ' He was drafted by the Bills in the seventh round of the 2014 NFL Draft.', ' He played college football at Miami.', ' Henderson attended Cretin-Derham Hall High School and originally signed a letter of intent to play college football at the University of Southern California, but was released from his commitment in July 2010 and eventually committed to the University of Miami.']], ['George Thomas (American football)', ['George Carroll Thomas, Jr. (March 4, 1928 – May 23, 1989) was an American football halfback and defensive back in the National Football League for the Washington Redskins and the New York Giants.', ' He was a standout high school basketball player, which led to his being recruited to play college basketball for Tulane University.', ' However, first year OU football coach, Jim Tatum, convinced him stay in Oklahoma and play college football at the University of Oklahoma.', \" Thomas was a standout for the Sooners, lettering in '46, '47,'48 and '49.\", ' He earned All-American status in 1949 List of Oklahoma Sooners football All-Americans.', ' Thomas graduated from OU with a degree in Business Administration in 1950.']], ['Charlie Spiller', ['Charlie Spiller (born October 18, 1983) is a former American football wide receiver.', ' He was signed by the Tampa Bay Buccaneers as a street free agent in 2008.', ' He played college football at Alcorn State.']], ['Baron Batch', ['Baron Batch (born December 21, 1987), self-styled \"The Artist\", a Pittsburgh-based entrepreneur and former American football running back who retired from the NFL in 2013.', ' He is known for his \"FREE\" art drops, where he posts pictures of giveaway paintings on Instagram and Twitter, leaving clues to their location.', ' He played college football at Texas Tech University.', ' Batch chose to play college football at Texas Tech University over offers from Northwestern University, Duke University, and New Mexico State University.', ' Batch is from Midland, Texas.', ' He is the owner and creator of Angry Man Salsa and creative director of Studio AM.', ' He is the brother of Brian Batch of the band Alpha Rev.']], ['Johnson Bademosi', ['Johnson Bademosi (born July 23, 1990) is an American football cornerback and special teamer for the New England Patriots of the National Football League (NFL).', ' He was signed by the Browns as an undrafted free agent in 2012.', ' He was a member of the football, rugby, and track and field teams at Gonzaga College High School and went on to play college football for Stanford University.']], ['Ross Travis', ['Ross John Travis (born January 9, 1993) is an American football tight end for the Kansas City Chiefs of the National Football League (NFL).', ' He played college basketball at Penn State and did not play college football.', ' He signed with the Chiefs in 2015.']], ['Alcorn State University', ['Alcorn State University (Alcorn) is a historically black comprehensive land-grant institution located northwest of Lorman, Mississippi in rural Claiborne County.', ' It was founded in 1871 by the Reconstruction-era legislature to provide higher education for freedmen.', ' It is the first black land grant college established in the United States.', ' The university is counted as a census-designated place and had a resident population of 1,017 at the 2010 census.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.797\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae16ea85542990adbacf790', 'answer': '31 July 1975', 'question': 'When did the UVF Mid-Ulster Brigade conducted the attack The Miami Showband killings?', 'supporting_facts': [['UVF Mid-Ulster Brigade', 0], ['UVF Mid-Ulster Brigade', 5], ['Miami Showband killings', 0]], 'context': [['The Miami Showband', ['The Miami Showband were one of the most successful and popular showbands in Ireland in the 1960s and 1970s.', \" Led at first by singer Dickie Rock, and later by Fran O'Toole, they had seven number one records on the Irish singles chart.\", ' In 1975 during The Troubles, when returning from a performance in County Down, Northern Ireland, three members of the band, Fran O\\'Toole, Tony Geraghty, and Brian McCoy, were killed in what became known as the \"Miami Showband massacre\".']], ['Andrew Robb and David McIlwaine killings', ['The Tandragee killings took place in the early hours of Saturday 19 February 2000 on an isolated country road outside Tandragee, County Armagh, Northern Ireland.', ' Two young Protestant men, Andrew Robb and David McIlwaine, were beaten and repeatedly stabbed to death in what was part of a Loyalist feud between the loyalist Ulster Volunteer Force (UVF) and their rivals, the breakaway Loyalist Volunteer Force (LVF).', ' The men were not members of any loyalist paramilitary organisation.', ' It later emerged in court hearings that Robb had made disparaging remarks about the killing of UVF Mid-Ulster Brigade leader Richard Jameson by an LVF gunman the previous month.', ' This had angered the killers, themselves members of the Mid-Ulster UVF, and in retaliation they had lured the two men to the remote lane on the outskirts of town, where they killed and mutilated them.']], ['Loyalist Volunteer Force', ['The Loyalist Volunteer Force (LVF) is a small Ulster loyalist paramilitary group in Northern Ireland.', ' It was formed by Billy Wright in 1996 when he and his unit split from the Ulster Volunteer Force (UVF) after breaking its ceasefire.', \" They had belonged to the UVF's Mid-Ulster Brigade and Wright had been the brigade's commander.\", ' In a two-year period from August 1996, the LVF waged a paramilitary campaign with the stated goal of combatting Irish republicanism.', ' During this time it killed at least 14 people in gun and bomb attacks.', ' Almost all of its victims were Catholic civilians who were killed at random.', ' The LVF called off its campaign in August 1998 and decommissioned some of its weapons, but in the early 2000s a loyalist feud led to a number of killings.', ' Since then, the LVF has been largely inactive, but its members are believed to have been involved in rioting and organized crime.', ' In 2015, the security forces stated that the LVF \"exists only as a criminal group\" in Mid-Ulster and Antrim.']], ['UVF Mid-Ulster Brigade', ['UVF Mid-Ulster Brigade formed part of the loyalist paramilitary Ulster Volunteer Force in Northern Ireland.', ' The brigade was established in Lurgan, County Armagh in 1972 by its first commander Billy Hanna.', ' The unit operated mainly around the Lurgan and Portadown areas.', ' Subsequent leaders of the brigade were Robin Jackson, known as \"The Jackal\", and Billy Wright.', ' The Mid-Ulster Brigade carried out many attacks, mainly in Northern Ireland, especially in the South Armagh area, but it also extended its operational reach into the Republic of Ireland.', ' Two of the most notorious attacks in the history of the Troubles were carried out by the Mid-Ulster Brigade: the 1974 Dublin and Monaghan bombings and the Miami Showband killings in 1975.', ' Members of the Mid-Ulster Brigade were part of the Glenanne gang which the Pat Finucane Centre has since linked to at least 87 lethal attacks in the 1970s.']], ['Harris Boyle', ['Harris Boyle (1953 – 31 July 1975) was an Ulster Defence Regiment (UDR) soldier and a high-ranking member of the Ulster Volunteer Force (UVF), a Northern Irish loyalist paramilitary organisation.', ' Boyle was implicated in the 1974 Dublin and Monaghan bombings, and took part in the attack at Buskhill, County Down when an armed UVF gang wearing British Army uniforms ambushed The Miami Showband at a bogus military checkpoint.', ' The popular Irish cabaret band was driving home to Dublin after a performance in Banbridge.', \" He was one of the two gunmen killed when the bomb they had loaded onto the band's minibus exploded prematurely.\", ' He is sometimes referred to as Horace Boyle.']], ['John Francis Green', ['John Francis Green (18 December 1946 – 10 January 1975), was a leading member of the North Armagh Brigade of the Provisional Irish Republican Army, holding the rank of Staff Captain and Intelligence Officer.', ' He was killed in a farmhouse outside Castleblayney, County Monaghan, by members of the Mid-Ulster Brigade of the Ulster Volunteer Force (UVF).', \" According to Secret Intelligence Service operative Captain Fred Holroyd, British Army Captain Robert Nairac was involved in Green's killing.\", \" Green's was one of the 87 killings attributed by the Pat Finucane Centre to the group of loyalist extremists known as the Glenanne gang.\", ' No one was ever prosecuted for the killing.']], ['Miami Showband killings', ['The Miami Showband killings (also called the Miami Showband Massacre) was an attack by the Ulster Volunteer Force (UVF), a loyalist paramilitary group, on 31 July 1975.', ' It took place on the A1 road at Buskhill in County Down, Northern Ireland.', \" Five people were killed, including three members of The Miami Showband, who were then one of Ireland's most popular cabaret bands.\"]], ['1991 Cappagh killings', ['The 1991 Cappagh killings was a gun attack by the loyalist Ulster Volunteer Force (UVF) on 3 March 1991 in the village of Cappagh, County Tyrone, Northern Ireland.', \" A unit of the UVF's Mid-Ulster Brigade drove to the staunchly republican village and shot dead three Provisional IRA volunteers and a Catholic civilian at Boyle's Bar.\"]], ['Wesley Somerville', ['William Wesley Somerville (c. 1941 – 31 July 1975) was a Northern Irish loyalist, who held the rank of lieutenant in the illegal Ulster Volunteer Force\\'s (UVF) Mid-Ulster Brigade during the period of conflict known as \"the Troubles\".', \" He also served as a member of the British state's legal Ulster Defence Regiment (UDR).\", ' Somerville was part of the UVF unit that ambushed the Irish cabaret band The Miami Showband at Buskhill, County Down, which resulted in the deaths of three of the bandmembers.', \" Somerville was killed, along with Harris Boyle, when the bomb they had loaded onto the band's minibus exploded prematurely.\", \" His brother, John James Somerville (a former UDR soldier) was one of the three men convicted of the murders of bandmembers Brian McCoy, Fran O'Toole and Tony Geraghty.\"]], ['Richard Jameson (loyalist)', [\"Richard Jameson (c. 1953 – 10 January 2000), was a Northern Irish businessman and loyalist, who served as the leader of the paramilitary Ulster Volunteer Force's (UVF) Mid-Ulster Brigade.\", ' He was killed outside his Portadown home during a feud with the rival Loyalist Volunteer Force (LVF), the breakaway organisation founded by former Mid-Ulster UVF commander Billy Wright after he and the Portadown unit of the Mid-Ulster Brigade were officially stood down by the Brigade Staff (Belfast leadership) in August 1996.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.798\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab1d983554299340b52540a', 'answer': 'Ghostbusters Spooktacular', 'question': 'The attraction at universal studios that was based on \"The Tonight Show\" replaced an attraction that replaced an attraction based on what movie?', 'supporting_facts': [['Twister...Ride it Out', 1], ['Race Through New York Starring Jimmy Fallon', 0]], 'context': [['Twister...Ride it Out', ['Twister...Ride It Out was a special effects attraction located at Universal Studios Florida, based on the 1996 film \"Twister\".', ' It was announced in 1997 and replaced the Ghostbusters Spooktacular attraction in the New York area of the park.', ' The attraction was hosted by actors Bill Paxton and Helen Hunt, who starred in the original film.', ' The attraction closed on November 2, 2015 and replaced with \"Race Through New York Starring Jimmy Fallon\", which opened on April 6, 2017.']], ['Race Through New York Starring Jimmy Fallon', ['Race Through New York Starring Jimmy Fallon is a 3D motion-simulator attraction at Universal Studios Florida based on Jimmy Fallon\\'s tenure at \"The Tonight Show\".', ' It opened on April 6, 2017.']], ['Despicable Me Minion Mayhem', ['Despicable Me Minion Mayhem is a 4-D computer-animated simulator ride attraction located at Universal Studios Florida, Universal Studios Hollywood, and Universal Studios Japan.', ' The attraction is based on Universal Studios and Illumination Entertainment\\'s 2010 animated film \"Despicable Me\" and its franchise and employs the use of 3-D HD digital animation.', \" The attraction replaced Jimmy Neutron's Nicktoon Blast (Before Jimmy Neutron's Nicktoon Blast, that space was The Funtastic World of Hanna-Barbera) in Florida, in Hollywood, and in Japan.\"]], ['Guardians of the Galaxy (Epcot Attraction)', ['Guardians of the Galaxy is an upcoming attraction to be built at Epcot within the Walt Disney World Resort.', ' It will be the third attraction based on a Marvel Comics property at Walt Disney Parks and Resorts after the Iron Man Experience at Hong Kong Disneyland and at Disney California Adventure.', ' It will be the second attraction based on Marvel\\'s \"Guardians of the Galaxy\" to be built at a Disney theme park and is also the first Marvel-themed attraction to be built at Walt Disney World.', ' It will replace the Universe of Energy pavilion, which closed on August 13, 2017.']], ['Jaws (ride)', ['Jaws is a theme park attraction at Universal Studios Japan.', ' Based upon the films of the same name.', ' The attraction places guests aboard tour boats for what should be a leisurely tour of Amity Harbor, but instead becomes a harrowing chase between the craft and a very determined great white shark.', ' Jaws is an expanded version of a famous scene on the long-running backlot tour at Universal Studios Hollywood, also inspired by the film, and can be found at Universal Studios Japan near Osaka, and formerly, at Universal Studios Florida in Orlando.']], ['Backdraft (attraction)', ['Backdraft is a fire special effects show at Universal Studios Japan, and formerly Universal Studios Hollywood theme park.', ' The show is based on the film of the same name.', ' Visitors can learn how the pyrotechnic effects were created and experience some of them first hand.', ' The Hollywood attraction was supposed to officially close after Labor Day 2009 to be replaced by in 2011 but remained open to appease the temporary closures of other attractions for annual refurbishment.', ' The Hollywood attraction officially closed on April 11, 2010.', ' The attraction remains part of the Osaka park.', ' A similar styled show demonstrating storm effects, Lights!', ' Camera!', ' Action!', ' Hosted by Steven Spielberg, opened in Universal Studios Singapore in 2010.']], ['Waterworld: A Live Sea War Spectacular', ['Waterworld: A Live Sea War Spectacular is an attraction based on the 1995 film \"Waterworld\" found at Universal Studios Hollywood (1995), Universal Studios Japan (2001), and Universal Studios Singapore (2010).', ' The original attraction opened at the same time as the film.', ' Although the film was considered a critical and financial disappointment, the show was highly praised, winning a 1996 Thea Award from the Themed Entertainment Association.', ' The attraction remains highly rated by park guests.']], ['T2 3-D: Battle Across Time', ['T2 3-D: Battle Across Time is an attraction at Universal Studios Florida and Universal Studios Japan, with a former location at Universal Studios Hollywood.', ' The version of the show at Universal Studios Hollywood closed in 2012 and the version at Universal Studios Florida will close in 2017.']], ['Revenge of the Mummy', ['Revenge of the Mummy, officially named Revenge of the Mummy: The Ride, is an enclosed roller coaster based on the \"Mummy\" film franchise, located at Universal Studios Florida, Universal Studios Hollywood, and Universal Studios Singapore, using linear induction motors (LIMs) to launch riders from a complete standstill to a top speed of between 40 and in a matter of seconds.', ' All Revenge of the Mummy roller coasters have a minimum passenger height requirement of 48 in .', ' Two versions of the attraction have the same track layout but different storylines, however the attraction at Universal Studios Hollywood has an original layout and storyline.', ' All three attractions are manufactured by Premier Rides, feature track switches by Dynamic Structures, and are themed by Universal Creative and ITEC Entertainment Corporation.', ' Some of the alternate features of the Singapore version were designed by Adirondack Studios .']], ['Fear Factor Live', ['Fear Factor Live is an attraction located at Universal Studios Florida, and previously, Universal Studios Hollywood.', ' Both attractions opened in Spring of 2005.', ' The Hollywood attraction was closed on August 14, 2008 to make way for , which opened in Spring 2009.', ' The Florida attraction began operating on a seasonal schedule in February 2009.', ' The attraction began running again on a full-time basis (except for Halloween Horror Nights considerations) in the Summer of 2010.', ' The attraction is based on the NBC television series \"Fear Factor\", and features theme park guests becoming contestants in various stunts inspired by the show.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.798\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab29956554299194fa9342d', 'answer': 'Mandalay Entertainment', 'question': 'The 1989 Batman film was produced by Jon Peters and a man that is the CEO of what company?', 'supporting_facts': [['Batman (1989 film)', 0], ['Peter Guber', 1]], 'context': [['Batman (score)', ['Batman: Original Motion Picture Score is the score album for the 1989 film \"Batman\" by Danny Elfman.', ' According to the \"Batman\" DVD Special Edition, Elfman said that producer Jon Peters was not sure about him as a composer until Tim Burton made him play the main titles.', ' Elfman admitted he was stunned when Peters announced that the score would be released on its own album, as releasing a separate score album for a film was something that was rarely done in the 1980s.', ' Elfman\\'s \"The Batman Theme\" went on to become an iconic piece.', ' It served as the basis for the theme music of \"\", which premiered in 1992, although this was later changed.', ' Some parts of the Elfman score are also heard in \"\", \"\" and \"\".', ' Parts are also played in the queue, and on the station platform of Batman the Ride at various Six Flags theme parks.']], ['Batman vs. Two-Face', ['Batman vs. Two-Face is an upcoming 2017 American animated direct-to-video superhero film produced by Warner Bros.', ' Animation and distributed by Warner Bros.', ' Home Entertainment.', ' It is a sequel to \"\".', ' It will premiere at the New York Comic Con on October 8, 2017 and will be followed by a digital release on October 10, 2017 and on DVD and Blu-ray on October 17, 2017.', ' Based on the 1960s \"Batman\" TV series, the film stars Adam West (in his final role before his death), Burt Ward and Julie Newmar reprising their roles of Batman, Robin and Catwoman from the series.', ' The film will show a tribute to the late Adam West.', ' This could also be the final time Warner Bros.', ' Animation would make a 1960s Batman film.']], ['Batman (1989 film)', ['Batman is a 1989 American superhero film directed by Tim Burton and produced by Jon Peters and Peter Guber, based on the DC Comics character of the same name.', ' It is the first installment of Warner Bros.\\' initial \"Batman\" film series.', ' The film stars Michael Keaton as Bruce Wayne/Batman and Jack Nicholson as The Joker, alongside Kim Basinger, Robert Wuhl, Pat Hingle, Billy Dee Williams, Michael Gough and Jack Palance.', \" The film takes place early in the title character's war on crime, and depicts a battle with his arch-nemesis the Joker.\"]], ['God Loves, Man Kills (album)', [\"God Loves, Man Kills was One King Down's second release but first full-length album.\", ' It was released on December 8, 1998, through Equal Vision Records.', ' Featuring new singer Jon Peters (who would eventually be replaced by original One King Down vocalist, Rob Fusco, this record saw One King Down attempt to merge funk and skiffle into their sound.', ' The results received mixed reactions.']], ['Batman Forever (disambiguation)', ['Batman Forever is a 1995 film in the Batman film series.']], ['Batman (album)', ['Batman is the eleventh studio album by American recording artist Prince and the soundtrack album to the 1989 film \"Batman\".', ' It was released on June 20, 1989 by Warner Bros.', ' Records.', \" As a Warner Bros. stablemate, Prince's involvement in the soundtrack was designed to leverage the media company's contract-bound talent as well as fulfill the artist's need for a commercial (if not critical) revival.\", ' The result was yet another multi-platinum successful cross-media enterprise by Warner Bros., in the vein of \"Purple Rain\".', ' The album was No. 1 on the \"Billboard\" albums chart for six consecutive weeks.', ' It has sold over eleven million copies worldwide.', ' In 2016, film critic Matt Zoller Seitz praised Prince\\'s songs and music videos for \"Batman\", more so than the film itself, stating that his songs \"suggest a goofy, perverse, sensuous, somewhat introverted Batman film that so far we\\'ve never gotten from anyone\", and arguing that Prince\\'s music videos \"are more psychologically perceptive than any of the Batman films.\"']], ['The Man Who Falls', ['\"The Man Who Falls\" is a 1989 comic book story by Dennis O\\'Neil and Dick Giordano.', \" It is an overview of Bruce Wayne's early life, including his parents' murder, his time spent traveling and training throughout the world, and his return to Gotham City to become Batman.\", ' Sixteen years later, the story became the structural basis for \"Batman Begins\", which rebooted the Batman film franchise in 2005.']], ['Crazy for You (Madonna song)', ['\"Crazy for You\" is a song by American singer Madonna from the soundtrack album to the 1985 film \"Vision Quest\".', ' It was released on March 2, 1985 by Geffen Records as the first single from the soundtrack.', ' The song appears remixed on the greatest hits compilation \"The Immaculate Collection\" (1990) and was re-released on February 24, 1991 by Sire Records to promote the album.', ' The song was also included on the ballads compilation \"Something to Remember\" (1995) and the greatest hits compilation \"Celebration\" (2009).', ' Producers Jon Peters and Peter Guber, along with music director Phil Ramone, decided to use Madonna after listening to her previous recordings, employing John Bettis and Jon Lind to write the song.', ' After reading the script of the film, Bettis and Lind wrote the song about the situation in which the lead characters meet at a nightclub.', ' Initial recording sessions did not impress Bettis and Lind, and they felt that \"Crazy for You\" would be dropped from the soundtrack.', ' However, a new version was recorded to their liking.']], ['Peter Guber', ['Howard Peter Guber (born March 1, 1942) is an executive, entrepreneur, educator, and author.', ' He is Chairman and CEO of Mandalay Entertainment.', ' Guber\\'s most recent films from Mandalay Entertainment include \"The Kids Are All Right\", \"Soul Surfer\", and \"Bernie\".', ' He has also produced \"Batman\", \"The Witches of Eastwick\", and \"Flashdance.\"', \" Guber's films have earned over $3 billion worldwide and 50 Academy Award nominations.\"]], ['Batman: Return of the Joker', [\"Batman: Return of the Joker, known in Japan as Dynamite Batman (ダイナマイトバットマン , Dainamaito Battoman ) , is a 1991 platform video game, the follow-up to Sunsoft's on the Nintendo Entertainment System.\", ' Unlike that game, which was based on the 1989 Batman film directed by Tim Burton, \"Return of the Joker\" is entirely self-contained and based more on the modern comic book iteration of Batman.', ' However, Batman mans the Batmobile from the 1989 film.', ' A remake of \"Return of the Joker\", titled Batman: Revenge of the Joker, was released on the Sega Genesis by Ringler Studios in 1992.', ' A Super NES version of \"Revenge of the Joker\" was in development, but never released.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.799\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a77ace855429967ab105223', 'answer': 'Tamaulipas', 'question': 'What entity with the capital city of Ciudad Victoria is a home to the northern grasshopper mouse?', 'supporting_facts': [['Northern grasshopper mouse', 1], ['Tamaulipas', 1]], 'context': [['Ciudad Victoria International Airport', ['General Pedro J. Méndez International Airport (IATA: CVM,\\xa0ICAO: MMCV) is an international airport located in Ciudad Victoria, Tamaulipas, Mexico.', ' It handles air traffic of the city of Ciudad Victoria.', \" It's operated by Aeropuertos y Servicios Auxiliares, a federal government-owned corporation.\"]], ['Universidad La Salle', ['Universidad La Salle (ULSA) is a private institution of higher education with 15 campuses in Mexico.', ' It is part of the Institute of the Brothers of the Christian Schools.', ' It offers high school, bachelor, master and Ph.D degrees.', ' It has had an expansion in the country, creating its own university national system.', ' Its main campus is located in Mexico City, and has a presence in Ciudad Obregon, Chihuahua, Gomez Palacio, Monterrey, Ciudad Victoria, Leon, Morelia, Pachuca, Ciudad Nezahualcóyotl, Cuernavaca, Puebla, Oaxaca, Cancun and Saltillo.']], ['Northern grasshopper mouse', ['The northern grasshopper mouse (\"Onychomys leucogaster\") is a North American carnivorous rodent of the family Cricetidae.', ' It ranges over much of the western part of the continent, from central Saskatchewan and central Washington to Tamaulipas in northeast Mexico.']], ['Radio Tamaulipas', ['Radio Tamaulipas is the state radio network of Tamaulipas, originating from studios in the capital of Ciudad Victoria and airing on nine FM and three AM transmitters in the state.']], ['Tamaulipas', ['Tamaulipas (] ), officially the Free and Sovereign State of Tamaulipas (Spanish: \"Estado Libre y Soberano de Tamaulipas\" ), is one of the 31 states which, with Mexico City, comprise the 32 Federal Entities of Mexico.', ' It is divided into 43 municipalities and its capital city is Ciudad Victoria.', ' The capital city was named after Guadalupe Victoria, the first President of Mexico.']], ['Ciudad Victoria', ['Ciudad Victoria (] ), is the capital city of the Mexican state of']], ['Mexican Federal Highway 85', ['Mexico\\'s Federal Highway 85 (\"Carretera Federal 85\") connects Mexico City with the U.S. border at Nuevo Laredo, Tamaulipas.', ' Highway 85 runs through Monterrey, Nuevo León; Ciudad Victoria, Tamaulipas; Ciudad Valles, San Luis Potosí; and Pachuca, Hidalgo.', ' It ends at the intersection of Highway 95 in the San Pedro area of Mexico City.', ' Highway 85 is the original route of the Pan-American Highway from the border to the capital as well as the Inter-American Highway.']], ['Victoria, Cabañas', ['Victoria is a municipality in the Cabañas department of El Salvador.', ' Ciudad Victoria is home to a community-based radio station, Radio Victoria.']], ['Soto la Marina, Tamaulipas', ['Soto la Marina is a town in Soto la Marina Municipality located in the Mexican state of Tamaulipas.', ' It was directly hit by Hurricane Alex in 2010.', ' It is located on the banks of the Soto la Marina river, just up river from the small ocean port of La Pesca, and downriver from Ciudad Victoria, the capital of the State of Tamaulipas.', ' 180 miles South of Brownsville, Texas, it is accessible from there via a highway in approximately 3 hours driving time.']], ['Mexican Federal Highway 83', ['Mexican Federal Highway 83 (\"Carretera Federal 83\") is a Federal Highway of Mexico.', ' The highway travels from its northern junction with Mexican Federal Highway 85 (25.2\\xa0km / 15.7\\xa0mi north of Ciudad Victoria) to Ignacio Zaragoza, Tamaulipas to the south at the junction with Mexican Federal Highway 81.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.799\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a736e0155429901807db04f', 'answer': 'McDonnell Douglas', 'question': 'Who is the manufacturer of the plane that shot down Jameel Sayhood?', 'supporting_facts': [['Jameel Sayhood', 0], ['McDonnell Douglas F-15 Eagle', 0]], 'context': [['Korean Air Lines Flight 902', ['On 20 April 1978, Soviet air defense shot down Korean Air Lines Flight 902 (KAL 902) near Murmansk, Soviet Union, after the civilian aircraft violated Soviet airspace and failed to respond to Soviet ground control and interceptors.', ' Soviet Air Defence Forces initially identified it as part of the US air reconnaissance force, which carried out thousands of flights along Soviet borders annually at the time.', ' Captain Alexander Bosov, pilot of the Sukhoi Su-15 that brought down Flight 902, saw Asian logogram characters on the tail of the Korean aircraft, and reported this to the ground control.', ' Despite this, Vladimir Tsarkov, commander of the 21st Soviet Air Defence Corps, ordered Bosov to take down the plane, as the plane failed to respond to repeated orders to land, and was approaching the Soviet border with Finland.', ' The Su-15 opened fire, forcing the plane to descend, and killing two of the 109 passengers and crew members aboard Flight 902.', ' The plane made an emergency landing on the frozen Korpiyarvi lake (not to be confused with the Korpijärvi lake) near the Finnish border.']], ['El Al Flight 253 attack', ['El Al Flight 253, was a Boeing 707 en route from Tel Aviv, Israel, to New York City, United States when it was attacked by two Palestinian terrorists as it was about to depart from a layover in Athens, Greece on December 26, 1968.', ' One passenger, Israeli Leon Shirdan, 50, of Haifa, a marine engineer, was shot dead.', ' He was survived by his wife and then 15-year-old daughter.', ' Two unidentified women were injured, one by a bullet, the other as she leaped from the jet when the door was opened.', ' The two terrorists were 19-year-old Naheb H. Suleiman, born in Tripoli, Libya, of Palestinian parents, and 25-year-old Mahmoud Mohammad Issa Mohammad, born in 1943 in Palestine.', ' They were members of the Lebanese-based militant organization Popular Front for the Liberation of Palestine The two Arabs dashed out of the transit lounge of Athens Airport just as the Israeli plane, parked 200 yards away, was preparing to take off.', ' The plane had flown in earlier from Tel Aviv.', ' Mahmoud Mohammed Issa Mohammed fired at the plane for more than a minute with a submachine gun, killing one; while the other threw two hand grenades, creating panic aboard the plane carrying 10 crew members and 41 passengers.', ' The two men were taken into custody by Greek authorities.', ' Mahmoud Mohammed Issa Mohammad, was sentenced to 17 years and 5 months behind bars.', ' He was freed after less than 4 months after another Palestinian terrorist group hijacked a Greek airliner and demanded his release.', ' Subsequently he successfully hid his terrorist past and emigrated to Canada.', ' Once Canadian authorities learned of his crime, a protracted extradition process culminated in his extradition to Lebanon in 2013.']], ['Shannon Rutherford', ['Shannon Rutherford is a fictional character played by Maggie Grace on the ABC drama television series \"Lost\", which chronicled the lives of the survivors of a plane crash in the South Pacific.', ' Shannon was introduced in the pilot episode as the stepsister of fellow crash survivor Boone Carlyle (Ian Somerhalder).', ' She was a series regular until her funeral in \"What Kate Did\".', ' For most of her time on the Island, she was unhelpful and spent much of her time sunbathing.', ' She formed a relationship with another survivor from the plane crash, Sayid Jarrah (Naveen Andrews).', ' Shannon was accidentally shot by Ana Lucia Cortez who mistakes her for an Other.']], ['Through the Viewfinder photography', ['Through the Viewfinder photography (TtV) is a photographic or videographic technique in which a photograph or video or motion picture film is shot with one camera through the viewfinder of a second camera.', ' The viewfinder thus acts as a kind of lens filter.', ' The most popular method involves using a digital camera as the image taking camera and an intact twin-lens reflex camera (TLR) or pseudo-TLR as the \"viewfinder\" camera.', ' TLRs typically have square waist-level viewfinders, with the viewfinder plane at 90 degrees to the image plane.', ' The image in a TLR viewfinder is laterally reversed, i.e. it is a mirror image.']], ['2007 Abkhazia plane downing incident', [\"The 2007 Georgia plane downing incident refers to the possible downing, by Georgia's anti-aircraft system, of a military plane that violated Georgia's air space on August 21, 2007.\", \" While it is still not confirmed by Georgia whether the plane was downed, Abkhazia's break-away government confirmed that a plane went down, but denies that it was shot down.\"]], ['Slam dunk', ['A slam dunk, also dunk or dunk shot, is a type of basketball shot that is performed when a player jumps in the air, controls the ball(s) above the horizontal plane of the rim, and scores by putting the ball directly through the basket with one or both hands.', ' It is considered a type of field goal; if successful, it is worth two points.', ' Such a shot was known as a \"dunk shot\" until the term \"slam dunk\" was coined by former Los Angeles Lakers announcer Chick Hearn.']], ['Pacific Southwest Airlines Flight 1771', ['PSA Flight 1771 was a British Aerospace 146-200A, registration N350PS, on a scheduled flight from Los Angeles, California to San Francisco.', ' On December 7, 1987, it crashed in Cayucos, California, as a result of a murder–suicide by one of the passengers.', ' All 43 passengers and crew aboard the plane died, five of whom, including the two pilots, were presumably shot dead before the plane crashed.', ' The man who caused the crash, David A. Burke, was a disgruntled former employee of USAir, the parent company of PSA.', ' A dramatization of the incident was portrayed on the TV documentary series \"Mayday\".']], ['Gus Winckel', ['Willem Frederick August (Gus) Winckel (3 November 1912 – 17 August 2013) was a Dutch military officer and pilot who flew for the Royal Netherlands East Indies Air Force (ML-KNIL) in World War II.', ' During the attack on Broome, Western Australia, on 3 March 1942, Winckel managed to land his plane full of refugees safely on the Broome airstrip just before the Japanese attack.', ' He then dismounted the plane\\'s machine gun and shot down one of the Japanese fighters, the only Allied \"kill\" during the attack.']], ['Jameel Sayhood', ['Captain Jameel Sayhood was an Iraqi pilot in the Gulf War who it is claimed managed to attain one of the few aerial victories by the Iraqi Air Force in his MiG-29B, before being shot down by USAF Captain Craig Underhill and Captain Cesar Rodriguez with their F-15Cs mere minutes after his air-air victory.']], ['McDonnell Douglas F-15 Eagle', ['The McDonnell Douglas F-15 Eagle is an American twin-engine, all-weather tactical fighter aircraft designed by McDonnell Douglas (now Boeing) to gain and maintain air supremacy in aerial combat.', \" Following reviews of proposals, the United States Air Force selected McDonnell Douglas' design in 1967 to meet the service's need for a dedicated air superiority fighter.\", ' The Eagle first flew in July 1972, and entered service in 1976.', ' It is among the most successful modern fighters, with over 100 victories and no losses in aerial combat, with the majority of the kills scored by the Israeli Air Force.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.800\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae779a555429952e35ea931', 'answer': 'Europe', 'question': 'Orson Welles made Filming the Trial, a making-of film of the production of The Trial, which was originally filmed where?', 'supporting_facts': [['Filming The Trial', 0], ['The Trial (1962 film)', 1]], 'context': [['The Trial (1962 film)', ['The Trial (1962) is a film directed by Orson Welles, who also wrote the screenplay based on the novel of the same name by Franz Kafka.', ' Filmed in Europe, Welles stated immediately after completing the film: \"\"The Trial\" is the best film I have ever made\".', ' The film begins with Welles narrating Kafka\\'s parable \"Before the Law\" to pinscreen scenes created by the artist Alexandre Alexeieff.', \" Anthony Perkins stars as Josef K., a bureaucrat who is accused of a never-specified crime, and Jeanne Moreau, Romy Schneider, and Elsa Martinelli play women who become involved in various ways in Josef's trial and life.\", \" Welles plays the Advocate, Josef's lawyer and the film's principal antagonist.\"]], ['Moby Dick (unfinished film)', ['Moby Dick is an unfinished film by Orson Welles, filmed in 1971.', ' It is not to be confused with the incomplete (and now lost) 1955 film Welles made of his meta-play \"Moby Dick—Rehearsed\", or with Moby Dick (1956 film), in which Welles played a supporting role.']], ['Filming The Trial', ['Filming \\'The Trial\\' is an unfinished making-of film by Orson Welles, made in 1981, which focuses on the production of his 1962 film \"The Trial\".']], ['Orson Welles Show (radio)', ['Orson Welles Show (1941–42), also known as The Orson Welles Theater, Orson Welles and his Mercury Theater and the Lady Esther Show (after its sponsor), is a live CBS Radio series produced, directed and hosted by Orson Welles.', ' Broadcast Mondays at 10 p.m. ET, it made its debut September 15, 1941.', ' Its last broadcast was February 2, 1942.']], ['George Schaefer (film producer)', ['George Schaefer (November 5, 1888, Brooklyn, New York – August 8, 1981) was a movie producer and once the president of RKO in 1941 when Orson Welles made his classic film \"Citizen Kane\".', ' Schaefer, a top executive at United Artists, was hired as president of RKO in 1938.', ' He was fired from RKO in 1942 because of the controversy surrounding it and Welles\\' second film \"The Magnificent Ambersons\".']], ['This is Orson Welles', ['This is Orson Welles is a 1992 book by Orson Welles (1915–1985) and Peter Bogdanovich that comprises conversations between the two filmmakers recorded over several years, beginning in 1969.', \" The wide-ranging volume encompasses Welles's life and his own stage, radio and film work as well as his insights on the work of others.\", \" The interview book was transcribed by Bogdanovich after Welles's death, at the request of Welles's longtime companion and professional collaborator, Oja Kodar.\", ' Welles considered the book his autobiography.']], ['Orson Welles Commentaries', ['Orson Welles Commentaries (1945–46) is an ABC radio series produced and directed by Orson Welles.', ' Featuring commentary by Welles, with reminiscences and readings from literature, the 15-minute weekly program aired Sunday afternoons at 1:15\\xa0p.m. ET beginning September 16, 1945.', ' Lear Radio sponsored the program through the end of June 1946 when it failed to find a larger audience.', ' The series was continued by ABC as a sustaining show through October 6, 1946.', ' \"Orson Welles Commentaries\" was the last of Welles\\'s own radio shows.']], ['Vienna (film)', [\"Vienna (also known as Orson Welles' Vienna or Spying in Vienna) is a 1968 short film directed by Orson Welles.\", \" It was originally produced as part of his abandoned television special, Orson's Bag, which was made for CBS; but in 1969, with the project close to completion, CBS withdrew their funding over Welles' long-running disputes with US authorities regarding his tax status.\", ' The film remained uncompleted.', ' Despite its name, \"Vienna\" freely mixes footage shot in Vienna, Zagreb, and in a Los Angeles studio.', ' The 8-minute segment was restored by the Munich Film Museum in 1999.']], ['Richard France (writer)', ['Richard France (born May 5, 1938) is an American playwright, author, and film and drama critic.', ' He is a recognized authority on the stage work of American filmmaker Orson Welles.', ' His publication, \"The Theatre of Orson Welles\", which received a CHOICE Outstanding Academic Book Award in 1979, has been called \"a landmark study\" and has been translated into Japanese.', ' His 1990 companion volume, \"Orson Welles on Shakespeare\" has been praised by Welles critics and biographers.']], ['The Orson Welles Almanac', ['The Orson Welles Almanac (also known as Radio Almanac and The Orson Welles Comedy Show) is a 1944 CBS Radio series directed and hosted by Orson Welles.', ' Broadcast live on the Columbia Pacific Network, the 30-minute variety program was heard Wednesdays at 9:30 p.m. ET January 26 – July 19, 1944.', ' The series was sponsored by Mobilgas and Mobiloil.', ' Many of the shows originated from U.S. military camps, where Welles and his repertory company and guests entertained the troops with a reduced version of \"The Mercury Wonder Show\".', ' The performances of the all-star jazz band that Welles brought together for the show were an important force in the revival of traditional New Orleans jazz in the 1940s.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.801\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab7bfe955429928e1fe38c2', 'answer': 'casting, job opportunities, and career advice', 'question': 'Erik Marcisak was named one of Top Ten \"Comedy Best Bets\" by a brand with special focus on what ?', 'supporting_facts': [['Erik Marcisak', 1], ['Backstage (magazine)', 0]], 'context': [['Eurogliders', ['Eurogliders are a band formed in 1980 in Perth, Western Australia, which included Grace Knight on vocals, Bernie Lynch on guitar and vocals, and Amanda Vincent on keyboards.', ' In 1984, Eurogliders released an Australian top ten album, \"This Island\", which spawned their No.\\xa02 hit single, \"Heaven (Must Be There)\".', ' \"Heaven\" also peaked at No.\\xa021 on the United States \"Billboard\" Mainstream Rock charts and appeared on the Hot 100.', ' Another Australian top ten album, \"Absolutely\", followed in 1985, which provided two further local top ten singles, \"We Will Together\", and \"Can\\'t Wait to See You\".', ' They disbanded in 1989, with Knight having a successful career as a jazz singer.', ' Australian rock music historian Ian McFarlane described Eurogliders as \"the accessible face of post-punk new wave music.', ' The band\\'s sophisticated brand of pop was traditional in its structure, but displayed the decidedly \\'modern veneer\\' (hip clothes, heavy use of synthesiser)\".', ' The band reformed in 2005 releasing a new album followed ten years later by their sixth album.']], ['Nucleopore filter', ['A nuclepore filter (brand name Nuclepore from Whatman, part of GE Healthcare) is a kind of filter in which holes a few micrometres in size have been created in a plastic (e.g. polycarbonate) membrane.', ' These filters are generally created by exposing the membrane to radiation that weakens the plastic and creates specific areas that can be removed by dousing the membrane in acid (or other chemicals).', ' The technique and patent were developed by Robert L. Fleischer, P. Buford Price, and Robert M. Walker as an outgrowth of their research on radiation effects in solids, with a special focus on materials exposed to energetic particles in space.', ' The technique allows for creating uniform holes of any desired diameter to allow even a virus to be filtered.']], ['Erik Marcisak', ['Erik Marcisak (pronounced \"Mar-See-Sack\") (born March 17, 1978 in Queens, NY), is an American writer, sketch comedy producer, actor and VIP customer of David Gagnon taxi, based in Charlottetown, PEI.', ' Erik Marcisak was named one of \"Backstage\"\\'s Top Ten \"Comedy Best Bets\" in 2005 for producing the controversial sketch comedy show \"Saturday Night Rewritten\", which used the previous night\\'s \"Saturday Night Live\" as a creative jumping-off point for an entirely new sketch show that was written, rehearsed, and performed within 8 hours the next day.', ' \"Saturday Night Rewritten\" ran in New York City from 2003-2006.']], ['Blackout Improv', ['Blackout Improv is an improvisational comedy theatre troupe in Minneapolis, Minnesota.', ' Founded in 2015, the cast is completely black.', ' Topics of monthly comedy performances include standard improv audience suggestions as well as a special focus on civil rights issues like police brutality, white privilege, and cultural appropriation.', ' Blackout Improv responded to the shooting of Jamar Clark as well as the acquittal of police officer Jeronimo Yanez after the shooting of Philando Castile.']], ['Stefan Kiesbye', ['Stefan Kiesbye is a German novelist and poet.', ' His first novel, \"Next Door Lived a Girl\" won the Low Fidelity Press Novella Award.', ' The German edition was a KrimiWelt Top Ten crime novel pick for four consecutive months.', ' The book has also been translated into Dutch, Spanish and Japanese.', ' His second novel,\"Your House Is on Fire, Your Children All Gone\" was a Top Ten pick of Oprah Magazine, made Entertainment Weekly’s Must List, and was named one of the best books of 2012 by \"Slate\" editor Dan Kois.', ' Kiesbye\\'s stories, essays, and reviews have appeared in \"The Wall Street Journal\", \"Publishers Weekly\", and the \"Coachella Review\", among others.']], ['Kaysie Lackey', ['Kaysie Lackey (born Brentwood, Tennessee, USA) is a food artist and cake decorating instructor based in Seattle, Washington.', ' As owner of The People\\'s Cake in Seattle, WA, she has been featured in wedding and cake magazines, including \"Brides\", Martha Stewart Weddings\", \"Modern Wedding Cakes\", \"Seattle Bride\", \"Seattle Metropolitan Bride and Groom\" and \"American Cake Decorating\".', ' In 2015 Kaysie was also profiled in The Wall Street Journal\\'s \"What\\'s In Her Bag?\"', '.', ' She is a frequent competitor on Food Network Challenge cake decorating competitions, having been featured on four different episodes, and winning three as of 2012.', ' She was also featured on Food Network\\'s \"Last Cake Standing\".', ' Kaysie teaches at cake decorating schools in the United Kingdom, United States, Mexico, Canada, Australia, Asia, Africa, South America, India,throughout Europe, and the Middle East.', ' The People\\'s Cake was named one of \"Brides\" magazine\\'s Top 100 Cake Decorators in the United States in 2013.', ' Kaysie was named one of \"Martha Stewart Wedding\\'s\" Top 63 Pastry Professionals in 2014 and \"Dessert Professionals\" magazine\\'s Top Ten Wedding Cake Decorators of North America in 2015.']], ['Backstage (magazine)', ['Backstage (aka Back Stage) is an entertainment-industry brand aimed at people working in film and the performing arts, with a special focus on casting, job opportunities, and career advice.']], ['Frank Wolf (adventurer)', ['Frank Wolf (born 1970) is a Canadian adventurer, filmmaker, writer and environmentalist.', ' He is known for films, feature magazine articles, and online columns that document wilderness expeditions around the world, with a focus on the Canadian North.', ' His expeditions include being the first to canoe across Canada in one season and cycling 2,000\\xa0km in winter on the Yukon River from Dawson to Nome.', ' His films include \"Wild Ones\", \"The Hand of Franklin\", \"Kitturiaq\", \"On the Line\", \"Mammalian\", and \"Borealis\", all of which broadcast on CBC\\'s \"documentary \"channel in Canada.', \" In 2012 he was named one of Canada's Top Ten Adventurers by Explore Magazine, and in 2015 he was named One of Canada's Top 100 Explorers by Canadian Geographic Magazine.\"]], ['Aidan McGrath', ['Aidan McGrath is an Irish youth activist.', \" He is the former President of Ireland's National Youth Organisation.\", \" He was twice elected to represent his Constituency of Fingal in Ireland's National Youth Parliament, Dáil na nÓg, and was Chairperson of both the Swords Youth Council and the Fingal Comhairle na nÓg.\", \" McGrath is a member of both Fingal County Council's General Strategic Policy Committee and the Planning Strategic Policy Committee.\", ' In 2010 he was named one of the top Youth Leaders in both the Republic of Ireland and Northern Ireland by the National Association of Principals and Deputy Principals and the General Teaching Council for Northern Ireland.', ' McGrath continues to maintain a public profile in the area of political activism, and in 2012 he was named one of the top ten outstanding young people of Ireland by Junior Chamber International.']], ['S&amp;Man', ['S&Man (pronounced as Sandman) is a 2006 pseudo-documentary film that examines the underground subculture of horror films.', \" It contains interviews with indie horror filmmakers and other horror experts, including Erik Marcisak, Bill Zebub, Fred Vogel, Carol J. Clover, and Debbie D., as well as a scripted plot that comes into focus in the film's second half.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.801\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a886886554299206df2b258', 'answer': 'international football competition', 'question': 'What is the international football competition for women that Liu Ailing played in the 1991,1995, 1999 editions?', 'supporting_facts': [['Liu Ailing', 0], [\"FIFA Women's World Cup\", 0]], 'context': [[\"Gambia women's national football team\", [\"The Gambia women's national football team represents the Gambia in international football competition.\", \" The team, however, has not competed in a match recognised by FIFA, the sport's international governing body, despite that organised women's football has been played in the country since 1998.\", \" The Gambia has two youth teams, an under-17 side that has competed in FIFA U-17 Women's World Cup qualifiers, and an under-19 side that withdrew from regional qualifiers for an under-19 World Cup.\", \" The development of a national team faces challenges similar to those across Africa, although the national football association has four staff members focusing on women's football.\"]], ['EAFF E-1 Football Championship (women)', ['EAFF E-1 Football Championship is an international football competition in East Asia for national teams of the East Asian Football Federation (EAFF).', \" The competition between women's national teams is held alongside men's competition.\"]], ['EAFF E-1 Football Championship', [\"EAFF E-1 Football Championship, known as the East Asian Football Championship from 2003 to 2010, and the EAFF East Asian Cup for the 2013 and 2015 editions, is a men's international football competition in East Asia for member nations of the East Asian Football Federation (EAFF).\", ' Before the EAFF was founded in 2002, the Dynasty Cup was held between the East Asian top four teams, and was regarded as the unofficial East Asian Championship.', ' There is a separate competition for both men (first held in 2003) and women (first held in 2005).', \" There was also a combined points competition in 2005, where the results of the men's and women's teams are added together (not including qualifiers).\"]], [\"FIFA Women's World Cup\", ['The FIFA Women\\'s World Cup is an international football competition contested by the senior women\\'s national teams of the members of \"Fédération Internationale de Football Association\" (FIFA), the sport\\'s global governing body.', \" The competition has been held every four years since 1991, when the inaugural tournament, then called the Women's World Championship, was held in China.\"]], ['Geri Donnelly', ['Geraldine \"Geri\" Donnelly (born 30 November 1965) is a Canadian former soccer player.', \" A midfielder, she represented Canada at the 1995 and 1999 editions of the FIFA Women's World Cup and was named Canadian Player of the Year in 1996 and 1999.\", \" Donnelly was part of the Canadian squad who won the 1998 CONCACAF Women's Championship.\", ' She was selected as a member of the Canadian Soccer Hall of Fame in 2014.']], ['1929–30 British Home Championship', ['The 1929-30 British Home Championship was an edition of the annual international football tournament played between the British Home Nations.', ' 1930 was the year in which the tournament finally gained a serious rival as the premier international football competition, with the inception of the 1930 FIFA World Cup, held in Uruguay.', ' The Home Nations were not however members of FIFA due to disputes over the growing professionalism in continental and South American football.', ' As a result, they were not able to attend and indicated that even if they were invited they would have no interest in attending, deeming foreign opposition too weak for serious contest.', ' It is interesting to speculate what would have happened had the Home Nations entered the tournament, especially as the tide of world football was changing against Britain.', ' The England team, which dominated the 1930 championship, had lost to Spain the year before in the first defeat by a foreign football team, and in the same year they only managed draws with Germany and Austria.', ' The Scottish side, which had won most of the previous ten championships, was likewise unprepared, only playing its first game outside the British Isles in 1929, and being heavily defeated on tour in 1931 by both the Austrians and the Italians.']], ['2016 COTIF Tournament', [\"The 2016 L'Alcúdia International Football Tournament is a football competition which took place in July and August 2016.\", ' The 2016 edition was the first to feature only international youth teams.', ' Previous editions have contained a mix of national selections and club selections.']], [\"Togo women's national football team\", ['The Togo women\\'s national football team (French: \"Équipe du Togo féminine de football\" ) represents the Togolese Republic in women\\'s international football competition since 2006.', ' Togo is managed by the Fédération Togolaise de Football (FTF), the governing body of football in Togo.', ' The team only played five FIFA-recognised matches, all in 2006.', ' Their current manager is Paul Zoungbede.', \" Togo's home stadium is the Stade de Kégué, located in Lomé.\"]], ['Liu Ailing', [\"Liu Ailing (; born June 2, 1967) is a Chinese former footballer who played for the China national team at the 1991, 1995 and 1999 editions of the FIFA Women's World Cup.\", ' She won a silver medal at the 1996 Atlanta Olympics and participated at the 2000 Sydney Olympics.', ' A playmaking midfielder, she played professional club football in Japan and the United States.']], [\"Women's Nordic Football Championship\", [\"Women's Nordic Football Championship was an international football competition contested by the women's national football teams of the Nordic countries.\", ' The tournament was held annually between 1974 and 1982.', ' Finland, Denmark and Sweden competed from the start, Norway joined the tournament in 1978.', ' Iceland and Faroe Islands did not take part at the competition.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.802\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae385df5542992f92d82305', 'answer': 'Maro Ajemian', 'question': 'Who was a pianist, Maro Ajemian or Richard C. Sarafian?', 'supporting_facts': [['Maro Ajemian', 0], ['Richard C. Sarafian', 0]], 'context': [['Disaster on the Coastliner', ['Disaster on the Coastliner is 1979 American made-for-television action drama film about a disgruntled railroad employee who attempts to cause a collision between two passenger trains.', ' It was directed by Richard C. Sarafian and starred Lloyd Bridges, Raymond Burr, Robert Fuller, Pat Hingle, E. G. Marshall, Yvette Mimieux, William Shatner, and Paul L. Smith.', ' It originally aired on \"The ABC Sunday Night Movie\" on October 28, 1979.']], ['Solar Crisis (film)', ['Solar Crisis is a 1990 Japanese-American science fiction film.', ' The screenplay was written by Joe Gannon and Tedi Sarafian (credited as Crispan Bolt), based on the novel \"Kuraishisu niju-goju nen\" by Takeshi Kawata, and directed by Richard C. Sarafian (credited as Alan Smithee).', ' The cast featured Tim Matheson as Steve Kelso, Charlton Heston as Adm. \"Skeet\" Kelso, Peter Boyle as Arnold Teague, Annabel Schofield as Alex Noffe, Corin Nemec as Mike Kelso and Jack Palance as Travis.', ' The executive producers were Takeshi Kawata and Takehito Sadamura, with FX cinematographer Richard Edlund and veteran sound editor James Nelson as its producers.']], ['Shadow on the Land', [\"Shadow on the Land, also known as United States: It Can't Happen Here, is a 1968 television film which aired on ABC.\", ' It was adapted from the Sinclair Lewis novel \"It Can\\'t Happen Here\" by Nedrick Young, and directed by Richard C. Sarafian.', ' The plot involves a President creating a fascist, totalitarian regime in the United States, and a resistance movement forming against it.']], ['Maro Ajemian', ['Maro Ajemian (July 9, 1921 – September 18, 1978) was an American pianist.', \" Ajemian's career in contemporary music got its impetus from her Armenian heritage; she became known as a contemporary pianist after performing the U.S. premiere of Aram Khachaturian's Piano Concerto, which she chose to play based on the fact that Khachaturian was Armenian.\"]], ['BoyBand (film)', ['BoyBand is a comedic feature film released in 2010 that depicts the fictional saga of the first ever American boy band.', ' The film stars Michael Copon, Robert Hoffman, Ryan Hansen, Ryan Pinkston, Ernest Phillips, Lorenzo Hooker III, Kurt Fuller, Ming-Na, Richard Riehle, and Tom Wright.', ' The film was shot entirely in the county of Worcester, Massachusetts and even features a fictional Worcester High School.', ' \"BoyBand\" was produced by Worcester-based Artigo/Ajemian films.', ' The film was written and directed by Jon Artigo, and produced by Andrea Ajemian.', ' Kaz Gamble produced and recorded the entire soundtrack.', ' The film is also known as \"BoyBand: Breakin\\' through in \\'82\".']], ['The Man Who Died Twice', ['The Man Who Died Twice is a 1958 American crime film directed by Joseph Kane and written by Richard C. Sarafian.', ' The film stars Rod Cameron, Vera Ralston, Mike Mazurki, Gerald Milton, Richard Karlan and Louis Jean Heydt.', ' The film was released on June 6, 1958, by Republic Pictures.']], ['Andy (film)', ['Andy is a 1965 dramatic film starring Norman Alden and written and directed by Richard C. Sarafian.', \" It was Sarafian' s first directing credit for a feature film.\"]], ['Richard C. Sarafian', ['Richard Caspar Sarafian (April 28, 1930 – September 18, 2013) was an American television and film director and actor.', ' He compiled a versatile career that spanned over five decades as a director, actor, and writer.', ' He is best known as the director of the 1971 film \"Vanishing Point\".']], ['Lolly-Madonna XXX', ['Lolly-Madonna XXX (a.k.a.', ' The Lolly-Madonna War) is a 1973 film directed by Richard C. Sarafian.', ' The film was co-written by Rodney Carr-Smith and Sue Grafton, based on the novel \"The Lolly-Madonna War\" by Grafton.']], ['Tedi Sarafian', ['Tedi Sarafian is an American screenwriter.', ' He was a co-writer of \"\" (2003).', ' He is the son of Richard C. Sarafian, and the brother of Richard Sarafian Jr. and Deran Sarafian and the nephew of Robert Altman.', ' He also is the co-owner of Barefoot Sound, manufacturer of high-end recording monitors.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.803\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab7f1315542991d322237d1', 'answer': 'Los Angeles', 'question': '\"Look At Us Now\" is a song by American DJs based in which city ?', 'supporting_facts': [['Look At Us Now', 0], ['Lost Kings', 0]], 'context': [['I Wonder If I Take You Home', ['\"I Wonder If I Take You Home\" is a song recorded by Lisa Lisa and Cult Jam with Full Force in 1984.', ' Record Producer Kenny Beck discovered the song in a \"discard bin\" at Personal Records while looking for songs to include on his debut album with the label.', ' He was so impressed that he created a compilation break-dancing album, \"CBS/SuzyQ,\" just to include the song.', ' He released the album in Europe on CBS Records, and it immediately gained popularity as a dance hit with club DJs there.', ' Soon American DJs began playing the song in the United States on Columbia Records.', ' After the song received heavy play from these DJs, \"I Wonder If I Take You Home\" reached No. 1 on the \"Billboard\" Hot Dance Club Play chart for one week in June 1985.', ' On other US charts, it peaked at No. 6 on the R&B chart and reached No. 34 on the Hot 100.', ' The single was certified as gold in the U.S. by the RIAA.', ' Overseas, it charted at No. 12 on the UK Singles Chart.']], ['Phone Down', ['\"Phone Down\" is a song recorded by American DJs Lost Kings and singer Emily Warren.', ' It was released as a single on October 7, 2016, via Disruptor Records and RCA Records.']], ['Look At Us Now', ['\"Look At Us Now\" is a song by American DJs Lost Kings.', ' It features singer Ally Brooke and rapper ASAP Ferg.', ' The song was remixed by Kaidro.']], ['It Began in Afrika', ['\"It Began in Afrika\" is a song by British electronic music duo The Chemical Brothers.', ' It was released as the first single from their fourth album \"Come with Us\" on 10 September 2001.', ' Originally named \"Electronic Battle Weapon 5\" and released for DJs as a white label in June 2001, \"It Began in Afrika\" became a hit in clubs and was renamed for its official release.', ' The song contains vocal samples from the track \"Drumbeat\" by American musician Jim Ingram.']], ['Lost Kings', ['Lost Kings is an American DJ duo consisting of Robert Abisi and Nick Shanholtz, based in Los Angeles.', ' The duo gained popularity beginning in 2014 through their official remixes for artists such as Imagine Dragons, Krewella, Halsey, Vance Joy, Echosmith, Rihanna, and Tori Kelly, and through their original EDM/progressive house music.', ' Managed by Disruptor Management, a joint venture at Sony Music Entertainment, they signed to Disruptor Records/RCA Records in October 2016.']], ['Let Me Hold You (Turn Me On)', ['\"Let Me Hold You (Turn Me On)\" is a song by American DJs Cheat Codes and Dutch DJ Dante Klein.', ' Released by Spinnin\\' Records on July 1, 2016, the song is largely a cover of Kevin Lyttle\\'s 2003 hit \"Turn Me On\", though it does include new lyrics.', ' It is the follow-up to Cheat Codes\\' internationally successful single \"Sex\".']], ['808 Audio', ['808 Audio is an American headphones, earphones, and wireless speakers company based in Hauppauge, New York.', ' It is a subsidiary of Voxx International.', ' The company specializes in manufacturing and making studio headphones targeted at consumers who identify with the EDM culture, DJs, and/or look for studio-quality sound.']], ['Feel Good (Gryffin and Illenium song)', ['\"Feel Good\" is a song by American DJs Gryffin and Illenium.', ' The song features the vocals of American singer Daya, and was released on March 3, 2017, through Darkroom Records and Interscope Records.']], ['Delirious (Prince song)', ['\"Delirious\" is a song by American musician Prince, from his 1982 album, \"1999\".', \" It was the album's third single, and Prince's second top 10 hit, reaching number 8 in the US during the fall of 1983.\", ' The success of the single was boosted by the runaway success of the previous single, \"Little Red Corvette\", and also because DJs often played the first three album tracks in sequence, which just happened to be the order of the singles released from the album.']], ['When I Look at You', ['\"When I Look at You\" is a song by American singer-songwriter and actress Miley Cyrus.', ' The song was written by Hillary Lindsey and John Shanks, and produced by Shanks.', ' It was released on March 26, 2010 by Hollywood Records as the second and final single from Cyrus\\' first EP, \"The Time of Our Lives\" (2009).', ' \"When I Look at You\" was featured on the 2010 romantic-drama film \"The Last Song\" and its corresponding soundtrack, being used to promote the film.', ' The song contains an instrumentation primarily based on piano, while its lyrics speak of a dream boy.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.805\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a89cc99554299669944a5ae', 'answer': 'late 19th and early 20th centuries', 'question': 'Wiener Werkstätte Style was part of the movement that took place during what eras?', 'supporting_facts': [['Wiener Werkstätte Style', 1], ['Modernism', 0]], 'context': [['Battle of Gully Hole Creek', ['The Battle of Gully Hole Creek was a battle that took place on July 18, 1742 (new style) between Spanish and British forces in the Province of Georgia, resulting in a victory for the British.', \" Part of a much larger conflict, known as the War of Jenkins' Ear, the battle was for control of St. Simons Island, the British fortifications of Fort Frederica and Fort St. Simons, and the strategic sea routes and inland waters they controlled.\", ' After the victory, the Province of Georgia established undisputed claim to the island, which is now part of the U.S. state of Georgia.', ' The better-known Battle of Bloody Marsh, a skirmish also won by the British, took place on the island the same day.']], ['2007 BA-CA-TennisTrophy', ['The 2007 BA-CA-TennisTrophy was a tennis tournament played on indoor hard courts.', ' It was the 33rd edition of the event known that year as the BA-CA-TennisTrophy, and was part of the International Series Gold of the 2007 ATP Tour.', ' It took place at the Wiener Stadthalle in Vienna, Austria, from October 7 through October 14, 2007.']], ['2008 Bank Austria-TennisTrophy', ['The 2008 Bank Austria-TennisTrophy was a tennis tournament played on indoor hard courts.', ' It was the 34th edition of the event known that year as the Bank Austria-TennisTrophy, and was part of the International Series Gold of the 2008 ATP Tour.', ' It took place at the Wiener Stadthalle in Vienna, Austria, from October 6 through October 12, 2008.']], ['Lists of protests against the Vietnam War', ['Protests against the Vietnam War took place in the 1960s and 1970s.', ' The protests were part of a movement in opposition to the Vietnam War and took place mainly in the United States.']], ['Modernism', ['Modernism is a philosophical movement that, along with cultural trends and changes, arose from wide-scale and far-reaching transformations in Western society during the late 19th and early 20th centuries.', ' Among the factors that shaped modernism were the development of modern industrial societies and the rapid growth of cities, followed then by reactions of horror to World War I.', ' Modernism also rejected the certainty of Enlightenment thinking, and many modernists rejected religious belief.']], ['2005 BA-CA-TennisTrophy', ['The 2005 BA-CA-TennisTrophy was a tennis tournament played on indoor hard courts.', ' It was the 31st edition of the event known that year as the BA-CA-TennisTrophy, and was part of the International Series Gold of the 2005 ATP Tour.', ' It took place at the Wiener Stadthalle in Vienna, Austria, from October 10 through October 16, 2005.']], ['Wiener Werkstätte Style', ['With the foundation of the Wiener Werkstätte in 1903, a new artistic style was born that came to be known as the Wiener-Werkstätte-Stil (literally, the Vienna Workshops Style).', ' Beginning with the 14th Exhibition of the Vienna Sezession in 1902, the radical distinctiveness of certain Viennese artists began to emerge, setting a foundation for the widespread Modernist movement.', ' Among the innovators was the Viennese architect Josef Hoffmann.', ' His cubist sculpture created in 1902 marked a break into independence for many Viennese artists.', ' His works from this period are especially remarkable when one considers that the term \"cubism\" only found its way into the art lexicon around 1907 to describe the work of Pablo Picasso.']], ['2006 BA-CA-TennisTrophy', ['The 2006 BA-CA-TennisTrophy was a tennis tournament played on indoor hard courts.', ' It was the 32nd edition of the event known that year as the BA-CA-TennisTrophy, and was part of the International Series Gold of the 2006 ATP Tour.', ' It took place at the Wiener Stadthalle in Vienna, Austria, from October 9 through October 15, 2006.']], ['2004 BA-CA-TennisTrophy', ['The 2004 BA-CA-TennisTrophy was a tennis tournament played on indoor hard courts.', ' It was the 30th edition of the event known that year as the BA-CA-TennisTrophy, and was part of the International Series Gold of the 2004 ATP Tour.', ' It took place at the Wiener Stadthalle in Vienna, Austria, from October 11 through October 17, 2004.']], [\"Woman's club movement\", [\"The woman's club movement was a social movement that took place throughout the United States.\", ' While women\\'s organizations had always been a part of United States history, especially in African-American communities, it wasn\\'t until the Progressive era that it came to be considered a \"movement.\"', ' The first wave of the club movement during the Progressive era was started by white, middle-class women and a second phase by African-American women.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.805\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac02835554299294b218f2e', 'answer': 'Manchester', 'question': 'Frank Lamson-Scribner was adopted by a family near which town in Kennebec County?', 'supporting_facts': [['Frank Lamson-Scribner', 0], ['Frank Lamson-Scribner', 1], ['Manchester, Maine', 0]], 'context': [['Winslow, Maine', ['Winslow is a town and census-designated place in Kennebec County, Maine, United States, along the Kennebec River.', ' The population was 7,794 at the 2010 census.']], ['Frank Lamson-Scribner', ['Franklin Pierce Lamson was born April 19, 1851 in Cambridgeport, Massachusetts.', ' His parents Joseph Sanborn and Eunice Ellen (Winslow) Lamson died when he was 3 years old and he was adopted by the Virgil Scribner family near Manchester, Maine.', ' He received preparatory education at Hebron Academy, Kents Hill School, and Coburn Classical Institute and graduated from Maine State College of Agriculture and Mechanic Arts in 1873.']], ['Kennebec, North Carolina', ['Kennebec is a small unincorporated community in southern Wake County, North Carolina along the border of Harnett County.', ' The community is situated along North Carolina Highway 55 and is the site of the Fuquay-Angier Airfield (Kennebec Flying Club).', ' Much of the area has been recently annexed by the Harnett County town of Angier.', ' Kennebec was named for Kennebec County, Maine .', ' The community was also a stop on the former Durham and Southern Railway.']], ['Two Cent Bridge', ['The Ticonic Footbridge, popularly known as the Two Cent Bridge, is a suspension bridge that spans the Kennebec River between the city of Waterville and the town of Winslow in Kennebec County, Maine.', ' It is one of the oldest surviving wire-cable steel suspension bridges and also is considered to be the last known extant toll footbridge in the United States.']], ['Jon A. Lund', ['Jon A. Lund (born November 6, 1928) is an American attorney and politician from Maine.', ' Lund, a Republican, served as Maine Attorney General from 1972-1975.', ' Prior to his time as the first full-time attorney general in Maine history, Lund was an assistant country attorney for Kennebec County, member of the Augusta City Council and two-time county attorney for Kennebec County.', ' He was also elected to the Maine House of Representatives (1965-1966; 1969-1972) and Maine Senate (1967-1968).']], ['Kennebec County, Maine', ['Kennebec County is a county located in the U.S. state of Maine, in the United States.', ' As of the 2010 census, the population was 122,151.', ' Its county seat is Augusta, the state capital.', ' The county was established on 20 February 1799 from portions of Cumberland and Lincoln Counties.', ' The name Kennebec comes from the Eastern Abenaki \"/kínipekʷ/\", meaning \"large body of still water, large bay.\"']], ['Manchester, Maine', ['Manchester is a town in Kennebec County, Maine, United States, located at .', ' The population was 2,580 at the 2010 census.', ' The southern part of the town bordering Cobbosseecontee Lake is a popular recreation spot in central Maine, and part of the Winthrop Lakes Region.', ' Manchester is included in the Augusta, Maine micropolitan New England City and Town Area.']], ['Kennebec County Courthouse', ['The Kennebec County Courthouse is located at 95 State Street in Augusta, Maine, the state capital and county seat of Kennebec County.', ' Built in 1829 and twice enlarged, it is one of the oldest examples of Greek Revival architecture in the state, and its earliest known example of a Greek temple front.', ' The building, which is now mostly taken up by county offices, was listed on the National Register of Historic Places in 1974.']], ['China Lake (Maine)', ['China Lake is a lake in Kennebec County, Maine.', ' Located northeast of the state capital of Augusta, China Lake is situated in the towns of China and Vassalboro.', ' China Lake has two large basins connected by a narrow neck.', ' The elongated eastern basin with an average depth of less than 30 ft is entirely within the town of China, and has an irregular shoreline heavily developed with residences and seasonal cottages.', ' The more nearly circular western basin extending into East Vassalboro is as deep as 85 ft , and shoreline development around the western basin has been discouraged to allow use as a water supply for Waterville and Winslow.', ' The western basin overflows into Outlet Stream in the town of Vassalboro.', ' Outlet Stream flows 7 mi north to discharge into the Sebasticook River in Winslow 1 mi upstream of the Kennebec River.']], ['Waterville, Maine', ['Waterville is a city in Kennebec County of the U.S. state of Maine, United States, on the west bank of the Kennebec River.', ' Home to Colby College and Thomas College, the population was 15,722 at the 2010 census.', ' Waterville is also the second city which makes up the \"Augusta-Waterville, ME Micropolitan Statistical Area\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.807\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abbee315542993f40c73c1d', 'answer': 'yes', 'question': 'Could you read both Bicycling and National Review', 'supporting_facts': [['Bicycling (magazine)', 0], ['National Review', 0]], 'context': [['National Review', ['National Review (NR) is an American semi-monthly conservative editorial magazine focusing on news and commentary pieces on political, social, and cultural affairs.', ' The magazine was founded by the author William F. Buckley Jr. in 1955.', ' It is currently edited by Rich Lowry.']], [\"Kate O'Beirne\", [\"Kate Walsh O'Beirne (September 23, 1949\\xa0– April 23, 2017) was the President of National Review Institute.\", ' She was the Washington editor of \"National Review\".', ' Her column, \"Bread and Circuses,\" covered Congress, politics, and U.S. domestic policy.']], ['Kevin D. Williamson', ['Kevin Daniel Williamson (born September 18, 1972) is the roving correspondent for \"National Review.\"', ' He is also the theater critic for \"The New Criterion\".', ' He was previously deputy managing editor at \"National Review\".']], ['Jim Geraghty', ['Jim Geraghty is a conservative blogger and regular contributor to \"National Review Online\" and \"National Review\".', ' In addition to writing columns for \"National Review\", Geraghty also blogs for National Review Online and is a former reporter for States News Service.']], ['National Review Board', ['The National Review Board (full name: National Review Board for the Protection of Children and Young People) is a committee created in 2002 by the United States Conference of Catholic Bishops in order to monitor the implementation of the Charter for the Protection of Children and Young People in the wake of the clerical abuse scandal in the United States.', ' The Board was also charged with investigating the scandal, which it did in part by commissioning the John Jay College to conduct a survey of Church records in order to define the nature and scope of the abuse committed by priests over the period between 1950 and 2002.', ' The results of that survey were released in 2004 in what has come to be known as the \"John Jay Report\" or \"Nature and Scope report.\"', ' In parallel with the John Jay College survey the Board conducted interviews with a variety of people, both inside and outside the Church, who were well placed to comment on the scandal, and on the basis of these interviews prepared a more broad-ranging report of its own.', ' Whereas the John Jay College report was (as intended) primarily a factual summary of the data collected in the College’s survey of Church records, the Board’s own report sought to interpret these data and its other findings in order to explain why the “epidemic” of clerical abuse had occurred and to identify the appropriate steps to avoid any repetition.']], ['Mark Krikorian', ['Mark Krikorian has been the executive director of the Center for Immigration Studies, a think-tank in Washington, D. C., since 1995.', ' The Center describes itself as an \"independent, non-partisan research organization\" in Washington, D. C., that examines and critiques the impact of immigration on the United States.', ' Animated by a \"pro-immigrant, low-immigration vision which seeks fewer immigrants, but a warmer welcome for those admitted\", the Center was established in 1985 to provide immigration research.', ' Krikorian is a regular contributor to the conservative publication \"National Review\", and is a regular participant at \"National Review Online\\'s\" \"The Corner.\"']], ['Review', ['A review is an evaluation of a publication, service, or company such as a movie (a movie review), video game (video game review), musical composition (music review of a composition or recording), book (book review); a piece of hardware like a car, home appliance, or computer; or an event or performance, such as a live music concert, play, musical theater show, dance show, or art exhibition.', \" In addition to a critical evaluation, the review's author may assign the work a rating to indicate its relative merit.\", ' More loosely, an author may review current events, trends, or items in the news.', ' A compilation of reviews may itself be called a review. \"', 'The New York Review of Books\", for instance, is a collection of essays on literature, culture, and current affairs. \"', 'National Review\", founded by William F. Buckley, Jr., is an influential conservative magazine, and \"Monthly Review\" is a long-running socialist periodical.']], ['Charles C. W. Cooke', ['Charles C. W. Cooke (born November 4, 1984) is the editor of \"National Review Online\".', ' He took the role over after Rich Lowry stepped down in June, 2016 (Lowry remains the editor-in-chief of \"National Review\").', ' Cooke is the author of \"The Conservatarian Manifesto\" and a frequent guest on HBO\\'s \"Real Time with Bill Maher\".', ' In addition to \"National Review\", he has written for the \"New York Times\", the \"Washington Post\", and the \"Los Angeles Times\".', ' Along with Kevin D. Williamson, he hosts the popular \"Mad Dogs and Englishmen\" podcast.']], ['Bicycling (magazine)', ['Bicycling is a cycling brand published by Rodale, Inc. in Emmaus, Pennsylvania.', ' \"Bicycling\" claims to be the world’s largest cycling magazine.']], ['The Human Life Review', ['The Human Life Review is a quarterly journal published by the Human Life Foundation since 1975.', ' It is devoted to explorations of life issues, primarily abortion, as well as neonaticide, medical genetics, prenatal testing, human cloning, fetal tissue experimentation, euthanasia and assisted suicide, and also publishes articles dealing with more general questions of family and society.', ' It was founded by James Patrick McFadden, formerly associate publisher of \"National Review\", who had also founded the Human Life Foundation, and is now edited by his daughter, Maria McFadden.', ' It was launched from the offices of \"National Review\", with the support of William F. Buckley.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.807\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a7a2dac5542990198eaf0b6', 'answer': 'Keisuke Kuwata', 'question': 'Who is younger, Keisuke Kuwata or Moya Brennan?', 'supporting_facts': [['Keisuke Kuwata', 0], ['Moya Brennan', 0]], 'context': [[\"Leo's Tavern\", ['Leo\\'s Tavern (Irish: \"Tábhairne Leo\" ) is a restaurant and pub in the Donegal Gaeltacht, best known as the home of music artists Clannad, Enya and Moya Brennan.', \" The pub opened in 1968 and held Irish traditional music sessions nightly, becoming the musical starting block for the children of Leo Brennan, the pub's founder.\", ' The current proprietor is Bartley Brennan, one of the youngest of the Brennan siblings.']], [\"My Match Is a Makin'\", [\"My Match Is A Makin' (English translation of opening track title) is a music album by Irish musicians Moya Brennan and Cormac de Barra.\", \" This is Moya's eighth studio album to be released.\", ' It was released on 14 April 2010 exclusively to concert goers on her Spring 2010 tour of the Netherlands.', ' The album was re-released to a wider audience under the title Voices & Harps.']], ['Keisuke Kuwata', ['Keisuke Kuwata (桑田 佳祐 , Kuwata Keisuke , born February 26, 1956) is a Japanese multi-instrumentalist, singer-songwriter, and frontman for the Southern All Stars, as well of his own solo band, the Kuwata band.', ' He has also done significant amount of scoring music for films.', ' He went to Aoyama Gakuin University.']], ['Two Horizons', ['Two Horizons is the fifth solo album by Irish singer Moya Brennan.', ' It is her first full-length release under the name \"Moya Brennan\", as opposed to \"Máire Brennan\".', ' The album was predominantly recorded in her home studio in Dublin and was nominated for a Grammy award.', ' The album was recorded between 2002–2003 and first became available on 23 October 2003.', \" It is also Brennan's first non-Christian album since her 1993 recording Misty Eyed Adventures.\", ' It is the most successful of her solo albums to date and the second to be Grammy-nominated.']], ['Keisuke Kuwata (album)', ['Keisuke Kuwata is the eponymous studio album recorded by Keisuke Kuwata, a frontman of the Japanese rock band Southern All Stars.', ' It was released by Taishita label under the Victor Entertainment in July 1988, shortly after the 10th anniversary of the band.', ' Aside from the 1982 live recording album \"Kamon Yuzo and Victor Wheels Live\" released under the pseudonym Yūzō Kamon, it was first Kuwata\\'s solo effort.']], ['No One Talks', ['\"No One Talks\" was the first commercial single taken from Grammy award-winning singer, Moya Brennan\\'s album \"Signature\" released the same year.', \" This was Moya's first single available to download from more than one online shop.\", ' The B-side to the single, the traditional Gaelic song \"Éirigh Suas a Stóirín (Rise Up My Love)\" was previously only available on the Germany-only album \"Óró - A Live Session\".', ' The cover shows a photograph by Mella Travers.']], ['Óró – A Live Session', ['Óró – A Live Session is a music album by Irish musician Máire Brennan, now known as \"Moya Brennan\".', ' Just before the start of the German tour, Moya and her band recorded a \"live session\" album to have available for fans on the tour.', \" She's tried to recreate some of the live feel with the band in the studio covering many of the songs she plays (and was about to play) in concert.\", ' The album will become more widely available subsequently, but originally was released in 2005.']], ['An Irish Christmas', ['An Irish Christmas is a music album by Irish musician Moya Brennan.', ' According to Moya, the idea for the album first came to her some time ago: \"I\\'ve been involved in number of other people\\'s Christmas projects in recent years,\" explains Moya, \"but I wanted to capture a truly Celtic Christmas feeling.\"', ' \"It\\'s always important to bring the meaning of Christmas to the fore.', ' It is the essence of what I believe in and the album offers both celebration and reflection on that familiar theme.\"']], ['Moya Brennan', ['Moya Brennan, born Máire Ní Bhraonáin (] ), also known as Máire Brennan (born 4 August 1952), is an Irish folk singer, songwriter, harpist, and philanthropist.', ' She is the older sister of Enya and Brídín Brennan.', ' She began performing professionally in 1970 when her family formed the band Clannad, and is considered as the \"First Lady of Celtic Music\".', ' Moya released her first solo album in 1992 called \"Máire\", a successful venture.', ' She has been nominated for two Grammys and has won an Emmy Award.', ' She has recorded music for several soundtracks, including \"Titanic\", \"To End All Wars\" and \"King Arthur\".']], [\"Ireland: Landscapes of God's Peace\", [\"Ireland: Landscapes of God's Peace is Máire Brennan's (Moya Brennan) 2000 book.\", ' It contains Celtic prayers, lyrics to Brennan\\'s \"Perfect Time\" album and her personal thoughts on Ireland, Christianity and Celtic culture.', ' The book comes in a set with the \"Perfect Time\" album, although it now considered a collectors item by Brennan\\'s fans.', ' Tynedale House Publishers print a limited number every few years.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.808\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab2da13554299194fa93542', 'answer': 'Connor', 'question': 'ho is the brother of the American transgender actor known for his work on MTV\\'s \"Faking It?\"', 'supporting_facts': [['Julia Fletcher', 3], ['Elliot Fletcher', 0]], 'context': [['Amanda Lepore', ['Amanda Lepore (born November 21, 1967) is an American transgender model, celebutante, singer, and performance artist.', ' The former Club Kid has appeared in advertising for numerous companies.', \" Lepore is also noted as a regular subject in photographer David LaChapelle's work, serving as his muse, as well as many other photographers, such as Terry Richardson and .\", ' She participated in LaChapelle\\'s \"Artists and Prostitutes 1985–2005\" exhibit in New York City, where she \"lived\" in a voyeuristic life-sized set.', ' Lepore has also released several singles, many written by and/or recorded with Cazwell.', ' In 2011, she released her debut studio album, \"I..', '.', 'Amanda Lepore\", on Peace Bisquit.']], ['Raffi Freedman-Gurspan', ['Raffi Freedman-Gurspan (born May 3, 1987 in Intibucá, Honduras) is an American transgender rights activist and the first openly transgender person to work as a White House staffer.', ' She was also the first openly transgender legislative staffer to work in the Massachusetts House of Representatives.', ' She currently serves as director of external relations at the National Center for Transgender Equality, based in Washington, DC.', ' She is a longtime advocate and public policy specialist on matters concerning human rights, gender, and LGBT people.']], ['Elliot Fletcher', ['Elliot Fletcher (born June 30, 1996) is an American transgender actor known for his work on the MTV comedy series \"Faking It\" and Freeform\\'s \"The Fosters\".']], ['Faking It (U.S. TV series)', ['Faking It is an American single-camera romantic comedy series that premiered on MTV on April 22, 2014, starring Rita Volk, Katie Stevens, Gregg Sulkin, Michael Willett and Bailey De Young.', ' The series was created by Dana Min Goodman and Julia Wolov.', ' Carter Covington developed the series and serves as the executive producer.', ' An eight-episode first season was ordered by MTV in October 2013.', ' MTV announced a ten episode second season set to premiere on September 23, 2014.', ' In August 2014, the show won a Teen Choice Award for \"Choice TV Breakout Show\".', ' In October 2014, MTV ordered 10 more episodes, meaning season 2 would have a total of 20 episodes.', \" The series features the first intersex main character on a television show, and included television's first transgender character played by a transgender actor.\"]], ['Stacey Hollywood', ['Stacey Hollywood is an American transgender woman, actor, model, and well-known LGBT nightclub personality.', ' In West Hollywood during the 1990s she became a prominent club promoter, hosting full-to-capacity nights at Club Arena, one of the largest nightclubs in Los Angeles.', \" She was featured on a popular 1998 house music club dance track with Club Arena's resident DJ Irene that was originally released on vinyl and later on a CD compilation.\"]], ['Kitana Kiki Rodriguez', ['Kitana Kiki Rodriguez is an American transgender actress.', ' She is best known for her role as Sin-Dee Rella in Sean Baker\\'s 2015 film \"Tangerine\".', ' The first Academy Award campaigns for openly transgender actresses supported by a film producer were launched for Rodriguez and Mya Taylor for \"Tangerine\".']], ['Julia Fletcher', ['Julia Fletcher (born 1958), sometimes credited as Julia DeMita, is an American voice actress whose throaty, articulate vocals have graced many popular animated works and video games.', ' Among her best-known roles are the Instructor (narrator) in \"The Animatrix: The Second Renaissance\", Elma and Yunalesca in \"Final Fantasy X\", Carmila in \"\", Scaphandra and Judy in \"Aeon Flux\", and Olmpias in the first four episodes of \"Reign: The Conqueror\".', ' She is married to voice actor John DeMita.', ' They have two sons, Connor (born 1995) and Elliot Fletcher (born 1996).', ' She also was voice cast for Orphan in \"Final Fantasy XIII\".']], ['Ho Ho Ho', ['Ho Ho Ho, also known as VH-1 Presents RuPaul: Ho Ho Ho, is a 1997 Christmas album and third overall by American singer and drag queen RuPaul.', ' It was released on October 28, 1997 by Rhino; it is RuPaul’s first album featuring Christmas music and serves as a follow-up to \"Foxy Lady\" (1996).', ' RuPaul co-produced the album with American music producers Fenton Bailey and Randy Barbato, who both work for the production company World of Wonder.', ' \"Ho Ho Ho\" consists of ten tracks, including eleven covers of Christmas standards and carols, and three original songs written by Joe Carrano and the singer.', ' The album was frequently referenced as an example of camp though RuPaul did specify that he recorded several more serious covers, specifically \"All Alone on Christmas\" and \"Hard Candy Christmas\".']], ['Creature (1999 film)', ['Creature is a 1999 documentary film that was directed by Parris Patton.', ' The film was released on June 3, 1999 and follows the life of American transgender actor, model, and club personality Stacey \"Hollywood\" Dean.']], ['Gregg Sulkin', ['Gregg Sulkin ( ; born 29 May 1992) is an English actor.', ' At age ten he made his film debut in the 2002 \"Doctor Zhivago\" mini-series.', ' He later landed the starring role in the 2006 British release \"Sixty Six\", and subsequently became known for appearing in the Disney Channel comedy series \"As the Bell Rings\" and \"Wizards of Waverly Place\".', ' In 2010, he starred in the Disney Channel television movie \"Avalon High\".', ' He also appeared in the television special \"\".', ' He starred on MTV\\'s show \"Faking It\" as Liam Booker from 2014 until its cancellation in 2016.', ' He also appeared on \"Pretty Little Liars\" as Ezra\\'s younger brother, Wesley \"Wes\" Fitzgerald.', ' In 2016, he starred in the role of Sam Fuller in the horror-thriller film, \"Don\\'t Hang Up\".']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.809\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae7b001554299540e5a5645', 'answer': 'most awarded female act of all-time', 'question': 'Brandy was handpicked by a producer that herself had been recognized by the \"Guinness World Records\" for what?', 'supporting_facts': [['Brandy filmography', 7], ['Whitney Houston', 0], ['Whitney Houston', 1]], 'context': [['George Hood (athlete)', ['George E. Hood is an ultra athlete, certified personal trainer (NESTA) and a Group-X instructor.', ' Hood also owns and operates a successful business known as Oceanside Paddleboard in Oceanside, CA.www.OceansidePaddleboard.com.', ' George holds 7 world records.', ' He has set a total of 6 Guinness World Records and one independent world record for the plank set in Beijing, China in June, 2014 which was certified in the media and by the Assist World Records organization in India.http://www.china.org.cn/wap/2014-06/24/content_32752425.htm George previously held the Guinness World Record for the prone hold, or plank at 3 hours, 7 minutes, and 15 seconds set on 20 April 2013.', 'http://www.thepostgame.com/blog/eye-performance/201304/george-hood-sets-planking-record-again']], ['Brandy filmography', ['As an actress, Brandy has appeared in feature films and television shows.', ' She made her television debut in 1993 in the ABC sitcom \"Thea\", as the daughter of a single mother (Thea Vidale).', ' Broadcast to low ratings, the series ran for only one season, but earned her a Young Artists Award nomination for Outstanding Youth Ensemble alongside her co-stars.', ' In 1996, her short-lived engagement on \"Thea\" led Brandy to star in her own show, the UPN-produced sitcom \"Moesha\", in which she played the title role of Moesha Mitchell, a Los Angeles girl coping with a stepmother as well as the pressures and demands of becoming an adult.', ' The program debuted on UPN in January 1996, and soon became their most-watched show.', ' The network decided to cancel the show after six seasons on the air, leaving it ending with a cliffhanger for a scrapped seventh season.', ' Brandy was awarded an NAACP Image Award for her performance.', ' In 1997, Brandy was hand-picked by producer Whitney Houston to play the title character in Rodgers and Hammerstein’s television version of \"Cinderella\" featuring a multicultural cast that also included Jason Alexander, Whoopi Goldberg, and Houston.', ' The two-hour \"Wonderful World of Disney\" special garnered an estimated 60 million viewers, giving the network its highest ratings in the time period in 16 years, and won an Emmy Award the following year.']], ['List of best-selling singles', ['According to \"Guinness World Records\", \"White Christmas\" (1942) by Bing Crosby is the best-selling single worldwide, with estimated sales of over 50 million copies.', ' The song recognized as \"the best-selling single of all time\" was released before the pop/rock singles-chart era and \"was listed as the world\\'s best-selling single in the first-ever \"Guinness Book of Records\" (published in 1955) and—remarkably—still retains the title more than 50 years later.\"', ' \"Guinness World Records\" states that double A-side charity single \"Candle in the Wind 1997\"/\"Something About the Way You Look Tonight\" (1997) by Elton John, a tribute to the late Diana, Princess of Wales, is \"the biggest-selling single since UK and US singles charts began in the 1950s, having accumulated worldwide sales of 33 million copies,\" making it the second-best-selling single of all time.']], ['Whitney Houston', ['Whitney Elizabeth Houston (August 9, 1963 – February 11, 2012) was an American singer, actress, producer, and model.', ' In 2009, \"Guinness World Records\" cited her as the most awarded female act of all-time.', ' Houston is one of the best-selling music artists of all-time, with 200 million records sold worldwide.', ' She released seven studio albums and two soundtrack albums, all of which have diamond, multi-platinum, platinum, or gold certification.', ' Houston\\'s crossover appeal on the popular music charts, as well as her prominence on MTV, starting with her video for \"How Will I Know\", influenced several African American women artists who follow in her footsteps.']], ['Guinness World Records Gone Wild', ['Guinness World Records Gone Wild, also known as \"Guinness World Records Unleashed\", is an American reality television series on truTV.', ' The series debuted on February 7, 2013 and is hosted by Dan Cortese.', \" The series' first season averaged more than 1.3 million viewers and ranked as one of ad-supported cable's Top 3 programs in the Thursday 8 p.m. timeslot with key adult and male demos.\", \" It was also cable's No.1 unscripted entertainment program in the timeslot with men 18-49 and adults 18-34.\", ' It was announced in April 2013 that truTV has ordered an additional ten episodes.', ' Season 2 premiered on November 7, 2013, and features a title change to \"Guinness World Records Unleashed\".']], ['Dave Farrow', ['David Andrew Farrow (born January 10, 1975) is a two-time Canadian Guinness World Record Holder for Most Decks of Playing Cards Memorized in a Single Sighting, entrepreneur, memory coach, speed reader and keynote speaker.', ' He is best known for winning the Guinness World Records for Most Decks of Playing Cards Memorized in a Single Sighting in 1996 and again in 2007 when he set out to reclaim his record after it was beaten in 2002.', ' The initial record was set at the Guinness World Records museum in Niagara Falls, Ontario, Canada while the latter was performed for Discovery Channel Canada at CTV Television Network studios.', ' Both records were accomplished under the controlled supervision of multiple cameras and multiple independent witnesses.']], ['Bao Xishun', ['Bao Xishun (also known as Xi Shun; born 1951) is a herdsman from Inner Mongolia, China, recognized by \"Guinness World Records\" as one of the world\\'s tallest living men.', \" On September 17, 2009, Turkey's Sultan Kösen overtook Bao Xishun as the tallest living man in the world, when he was measured by Guinness World Records, standing 2.51 m tall.\"]], ['Guinness World Records – Ab India Todega', ['Guinness World Records – Ab India Todega (English: \"Guinness World Records – Now India will Break\") is a reality TV show based on the Guinness Book of World Records.', ' The show, which was hosted by Preity Zinta and Shabbir Ahluwalia, premiered on 18 March 2011 to an audience measurement of 3.3 rating points.', ' Each episode presents different individuals trying to break official world records.']], ['Twin Galaxies', ['Twin Galaxies is an American organization that tracks \"retro\" and \"old-school\" video game world records and conducts a program of electronic-gaming promotions.', ' It operates the Twin Galaxies website and publishes the \"Twin Galaxies\\' Official Video Game & Pinball Book of World Records\", with the Arcade Volume released on June 2, 2007.', ' \"The Guinness World Records - Gamers Edition 2008\" was released in March, 2008 in conjunction with Twin Galaxies, who Guinness World Records considers to be the official supplier of verified world records to the annual volume.']], ['Ashrita Furman', ['Ashrita Furman (born Keith Furman, September 16, 1954, Brooklyn, New York) is a Guinness World Records record-breaker.', ' As of 2017, Furman has set more than 600 official Guinness Records and currently holds 200 records, including the record for holding the most Guinness world records.', ' He has been breaking records since 1979.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.810\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5abc3d215542993a06baf8a5', 'answer': 'Acharacle', 'question': 'Loch Shiel is part of the river that drains into the sea near the castle that sits west of what town?', 'supporting_facts': [['Loch Shiel', 1], ['Castle Tioram', 0], ['Castle Tioram', 1]], 'context': [['Angle, Pembrokeshire', ['Angle (Welsh: \"Angl\" ) is a village and community on a narrow peninsula on the very south-west tip of Wales in Pembrokeshire.', \" It has two public houses, a school, post office, a castle, St Mary's church and a sandy beach to the west of the village.\", ' The nearest railway station is Pembroke, from where there is a bus link.', ' The Angle lifeboat received silver medals in 1878 for rescuing the crew of the \"Loch Shiel\" from rocks near Thorn Island.', ' The ship had been carrying a cargo of whisky and beer.']], ['Polloch', ['Polloch is a remote hamlet, located at the north shore of the River Polloch, in an inlet that flows into Loch Shiel, in Inverness-shire, Scottish Highlands and is in the Scottish council area of Highland.']], ['The Rough Bounds', ['The Rough Bounds (Scottish Gaelic: \"Na Garbh Chriochan\" ), in the Scottish Highlands, is the area of West Inverness-shire bounded by Loch Hourn, Loch Shiel, and Loch Moidart, consisting of the districts of Knoydart, North Morar, Arisaig and Moidart.', ' The area is famous for its wildness and inaccessibility and remains very sparsely populated.']], ['Sgùrr Ghiubhsachain', ['Sgùrr Ghiubhsachain is a mountain in the Lochaber area in the west of Scotland.', ' Its summit is the highest point in a group of mountains that stand south of Glenfinnan, to the south east of the northern part of Loch Shiel.', ' It is considerably lower than the nearby Nevis range, but it is a long way from a public road.', ' Its slopes are steep and rugged on all sides and are devoid of paths.', ' Despite easy access to the trail head from the road from Fort William to Mallaig, an individual or party that climbs this mountain may be alone there, even on a fine day in the summer.']], ['Shiel Bridge', ['Shiel Bridge is a village on the south east mouth of Loch Duich and confluence of the small loch in Loch Shiel and the River Shiel, in Lochalsh, Scottish Highlands and is in the council area of Highland.']], ['River Shiel', ['The River Shiel (Scottish Gaelic: Abhainn Seile) is a four kilometre long river in Acharacle, Highland.', ' It flows out of the Loch Shiel into the sea at Dorlin.']], ['Castle Tioram', ['Castle Tioram ( ) (Scottish Gaelic: \"Caisteal Tioram\" , meaning \"dry castle\") is a ruined castle that sits on the tidal island Eilean Tioram in Loch Moidart, Lochaber, Highland, Scotland.', ' It is located west of Acharacle, approximately 80 km from Fort William.', ' Though hidden from the sea, the castle controls access to Loch Shiel.', ' It is also known to the locals as \"Dorlin Castle\".', ' The castle is a scheduled monument.']], ['Loch Shiel', ['Loch Shiel (Scottish Gaelic: Loch Seile) is a 17+1/2 mi freshwater loch, 120 m (393 ft) deep, situated 12.4 miles west of Fort William in Lochaber, Highland, Scotland.', ' Its nature changes considerably along its length, being deep and enclosed by mountains in the north east and shallow surrounded by bog and rough pasture in the south west, from which end the 4 km River Shiel drains to the sea in Loch Moidart near Castle Tioram.']], ['Moidart', ['Moidart ( , Scottish Gaelic: Mùideart ] ) is part of the remote and isolated area of Scotland, west of Fort William, known as the Rough Bounds.', ' Moidart itself is almost surrounded by bodies of water : Loch Shiel cuts off the eastern boundary of the district (along a south-south-west to north-north-east line), and continues along part of the southern edge; the remainder of the southern edge is cut off by Loch Moidart; the north is cut off by Loch Morar and Loch Ailort.']], ['Glenfinnan Viaduct', ['The Glenfinnan Viaduct is a railway viaduct on the West Highland Line in Glenfinnan, Inverness-shire, Scotland.', ' Located at the top of Loch Shiel in the West Highlands of Scotland, the viaduct overlooks the Glenfinnan Monument and the waters of Loch Shiel.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.810\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a877df15542993e715abf7a', 'answer': 'yes', 'question': 'Are David Schwimmer and James Wan in the same industry?', 'supporting_facts': [['David Schwimmer', 0], ['James Wan', 0]], 'context': [['The Conjuring 2', ['The Conjuring 2 is a 2016 American horror film directed by James Wan.', ' The screenplay is by Chad Hayes, Carey W. Hayes, Wan and David Leslie Johnson.', ' It is the sequel to 2013\\'s \"The Conjuring\" and the third installment in \"The Conjuring\" series.', ' Patrick Wilson and Vera Farmiga reprise their roles as paranormal investigators and authors Ed and Lorraine Warren from the first film.', ' The film follows the Warrens as they travel to England to assist the Hodgson family, who are experiencing poltergeist activity at their Enfield council house in 1977 which later became referred to as the Enfield Poltergeist.']], ['Ross Geller', ['Dr. Ross Eustace Geller, Ph.D., is a fictional character from the NBC sitcom \"Friends\", portrayed by David Schwimmer.', ' Ross is considered by many to be the most intelligent member of the group and is noted for his goofy, pathetic but lovable demeanor.', ' He is a compulsive liar to avoid arguments or situations with conflict, often leading to an arcing storyline within a show.', ' His relationship with Rachel Green was included in \"TV Guide\"\\' s list of the best TV couples of all time, as well as \"Entertainment Weekly\"\\' s \"30 Best \\'Will They/Won\\'t They?\\'', ' TV Couples\".', ' \\xa0Kevin Bright, one of the executive producers of the show had worked\\xa0with Schwimmer before, so the writers were already developing Ross’s character in Schwimmer’s voice.', ' And hence, Schwimmer was the first person to be cast on the show.']], ['Aquaman (film)', ['Aquaman is an upcoming American superhero film based on the DC Comics character of the same name, distributed by Warner Bros.', ' Pictures.', ' It is intended to be the sixth installment in the DC Extended Universe (DCEU).', ' The film is being directed by James Wan, with a screenplay by Will Beall, from a story by Wan and Geoff Johns, and stars Jason Momoa, Amber Heard, Patrick Wilson, Willem Dafoe, Yahya Abdul-Mateen II, Temuera Morrison, Dolph Lundgren, and Nicole Kidman.']], ['The Pilot (Friends)', ['The Pilot, also known as \"The One Where Monica Gets a Roommate\" , was the first episode of the American situation comedy series \"Friends\", premiered on NBC (National Broadcasting Company) on September 22, 1994.', ' It was written by series creators David Crane and Marta Kauffman, and directed by James Burrows.', \" The pilot introduces six friends who live and work in New York City; Monica (Courteney Cox) sleeps with a wine seller after their first date but is horrified to discover he tricked her into bed; her brother Ross (David Schwimmer) is depressed after his lesbian ex-wife moves her things out of their apartment; Monica's old schoolfriend Rachel (Jennifer Aniston) moves in with Monica after running out of her wedding; and their friends, Joey, Chandler, and Phoebe (Matt LeBlanc, Matthew Perry, and Lisa Kudrow), respectively offer them each support and advice.\"]], ['Saw (2003 film)', ['Saw (retrospectively also known as Saw 0.5) is an Australian short subject horror film, released in 2003.', ' It was directed by James Wan and written by Wan and Leigh Whannell, the latter also starring in it.', ' It was originally used to pitch their script for a full-length feature film of the same name to various studios and actors.', ' The full-length film was eventually made in 2004.', ' The short film later became a scene in \"Saw\", with Shawnee Smith as Amanda Young wearing the Reverse Bear Trap device instead of David.', ' The original short can be viewed on the second disc of \"Saw: Uncut Edition\".']], ['Saw (franchise)', ['Saw is an American horror franchise distributed by Lionsgate, produced by Twisted Pictures and created by James Wan and Leigh Whannell, that consists of eight feature films and additional media.', ' In 2003, Wan and Whannell made a short film to help pitch as a potential feature film.', ' This was successfully done in 2004 with the release of the first installment at the Sundance Film Festival.', ' It was released theatrically that October.', ' The sequels were directed by Darren Lynn Bousman, David Hackl, and Kevin Greutert, and were written by Wan, Whannell, Bousman, Patrick Melton, and Marcus Dunstan, and were released subsequently every October, on the Friday before Halloween, between 2004 and 2010.', ' Both of the creators remained with the franchise as executive producers.', ' On July 22, 2010, producer Mark Burg confirmed that the seventh film, \"Saw 3D\", is the final installment of the series.', ' Lionsgate reportedly expressed interest in continuing the franchise in 2012 with a reboot.', ' However, in November 2013, it was reported that they were in active development of a sequel.']], ['James Wan', ['James Wan (born 27 February 1977) is a Malaysian-Australian film director, screenwriter, and producer.']], ['Saw (2004 film)', ['Saw is a 2004 American horror film directed by James Wan.', \" It is Wan's feature film directorial debut.\", ' The screenplay, written by Leigh Whannell, is based on a story by Wan and Whannell.', ' The film stars Cary Elwes, Danny Glover, Monica Potter, Michael Emerson, Ken Leung, Tobin Bell and Leigh Whannell.', ' In the film, Elwes and Whannell portray two men who awake to find themselves chained in a large dilapidated bathroom, with one being ordered to kill the other or his family will die.', ' It is the first installment in the \"Saw\" franchise.']], ['Trust (2010 film)', ['Trust (stylized as trust_) is a 2010 American drama thriller film directed by David Schwimmer and based on a screenplay by Andy Bellin and Robert Festinger, and an uncredited story by Schwimmer.', ' It stars Viola Davis, Clive Owen, Catherine Keener, Jason Clarke, and Liana Liberato.']], ['David Schwimmer', ['David Lawrence Schwimmer (born November 2, 1966) is an American actor, director, and producer.', ' He was born in Flushing, Queens, New York, and his family moved to Los Angeles when he was 2.', ' He began his acting career performing in school plays at Beverly Hills High School.', ' In 1988, he graduated from Northwestern University with a Bachelor of Arts in theater and speech.', ' After graduation, Schwimmer co-founded the Lookingglass Theatre Company.', ' For much of the late 1980s, he lived in Los Angeles as a struggling, unemployed actor.']]], 'type': 'comparison', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n",
      "\u001b[32m2025-12-09 18:10:18.811\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ae0ba9055429924de1b715c', 'answer': 'December 1993', 'question': 'Tony Leung Ka-fai is a four time winner of the award that has been on offer since what month?', 'supporting_facts': [['Tony Leung Ka-fai', 0], ['Hong Kong Film Award', 0]], 'context': [['Castrol Six Hour (New Zealand)', ['The New Zealand Castrol Six Hour Race was an endurance motorcycle race that was held for fifteen years at Manfeild, Palmerston North from 1974 to 1988.', \" Many of New Zealand's top international riders competed at the Six Hour race; winners included Dave Hiscock (5-time winner), Neil Chivas (4-time winner), Graeme Crosby (3-time winner), Aaron Slight (2-time winner), and Ginger Molloy (first winner).\", ' The race was characterised by its exclusive use of unmodified standard production motorcycles and a running start where the racers had to run to their machines before they could start.', ' There was special provisions that the machines were no older than 3 years old, which was a boon to the motorcycle industry in New Zealand at the time which saw a dramatic rise in the sale of Motorcycles after these events.']], ['Tony Leung Ka-fai', ['Tony Leung Ka-fai (; born 1 February 1958) is a Hong Kong actor who is a four-time winner of the Hong Kong Film Award.']], ['The Christ of Nanjing', ['The Christ Of Nanjing () is a 1995 erotic romantic drama film directed by Tony Au, starring Tony Leung Ka-fai and Yasuko Tomita.', ' The film is based on the work of famed Japanese novelist Akutagawa Ryunosuke.', ' Tomita won the award for best actress at 1995 Tokyo International Film Festival for her performance in the film.']], ['Tom, Dick and Hairy', ['Tom, Dick and Hairy is a 1993 Hong Kong romantic comedy film directed by Lee Chi-ngai and Peter Chan and starring Tony Leung Chiu-wai, Tony Leung Ka-fai and Lawrence Cheng as the three titular protagonists.']], ['Eye in the Sky (2007 film)', ['Eye in the Sky () is a 2007 Hong Kong crime film starring Simon Yam, Tony Leung Ka-fai and Miss Hong Kong pageant winner Kate Tsui in her film debut.', ' Yam and Tsui play surveillance operatives on the trail of a gang of professional robbers led by Chan Chong-Shan (Leung).', ' The title is derived from the casino surveillance tech \"eye in the sky\".', ' It marks the directorial debut of Yau Nai-hoi, a long-time screenwriter for films directed by Johnnie To, who co-produced the film with his production company Milkyway Image.', ' \"Eye in the Sky\" premiered as an Official Selection at the 2007 Berlin International Film Festival, and as an Opening Film at the Hong Kong International Film Festival.', ' It was released in Hong Kong on 21 June 2007.']], ['The Twins Effect II', ['The Twins Effect II is a 2004 Hong Kong action fantasy film directed by Corey Yuen and Patrick Leung.', ' The film is a sequel to \"The Twins Effect\" (2003), but has a completely different story from the first film.', ' It starred Charlene Choi and Gillian Chung of Cantopop duo Twins in the leading roles.', ' Co-stars include Donnie Yen, Daniel Wu, Edison Chen, Wilson Chen, Tony Leung Ka-fai, Qu Ying, Fan Bingbing and Jim Chim.', ' Jackie Chan also makes a cameo appearance, along with his son Jaycee Chan who is in his debut.', \" The film's original English working title was Huadu Chronicles: Blade of Rose and its US DVD release title is Blade of Kings.\"]], ['Always Be the Winners', ['Always Be the Winners is a 1994 Hong Kong comedy film directed by Jacky Pang and starring Tony Leung Chiu-Wai, Tony Leung Ka-fai, Eric Tsang, Sandra Ng, Ekin Cheng and Charine Chan.', ' The film was released during the Chinese New Year period of 1994 to celebrate the holidays.']], ['Hong Kong Film Award', ['The HKFA, incorporated into Hong Kong Film Awards Association Ltd. since December 1993, are currently managed by a board of directors, which consists of representatives from thirteen professional film bodies in Hong Kong.', ' Voting on eligible films for the HKFA is conducted January through March every year and is open to all registered voters, which include local film workers as well as critics, and a selected group of adjudicators.']], ['Cold War (film)', ['Cold War is a 2012 Hong Kong police thriller film directed by Sunny Luk and Longman Leung, starring Aaron Kwok and Tony Leung Ka-fai, and guest starring Andy Lau.', ' The film was selected as the opening film at the 17th Busan International Film Festival and released in Hong Kong, Macau, and mainland China on 8 November 2012.']], [\"He Ain't Heavy, He's My Father\", [\"He Ain't Heavy, He's My Father, also known as He Ain't Heavy, He's My Brother (新難兄難弟), is a 1993 Hong Kong comedy-drama film directed by Peter Chan and Lee Chi-ngai and starring Tony Leung Ka-fai, Tony Leung Chiu-wai, Carina Lau, Anita Yuen and Lawrence Cheng.\", ' The title is a pun of \"He Ain\\'t Heavy, He\\'s My Brother\", a hit single for The Hollies in 1969 and for Neil Diamond in 1970.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.812\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ab6b59f5542995eadef0065', 'answer': 'Gospel Starlighters', 'question': 'Lost Someone was written by James Brown and an American R&B/soul singer who began his career in 1952 as a member of what gospel group?', 'supporting_facts': [['Lost Someone', 0], ['Lost Someone', 1], ['Bobby Byrd', 0], ['Bobby Byrd', 1]], 'context': [['Hearts of Stone', ['\"Hearts of Stone\" is an American R&B song.', ' It was written by Eddie Ray and Rudy Jackson, a member of the San Bernardino, California-based rhythm and blues vocal group the Jewels (no relation to the female Jewels group from Washington, DC) which first recorded it for the R&B label in 1954.', ' The Jewels began as a gospel group, then became the Marbles, recording for the Lucky label out of Los Angeles.']], ['Respect Yourself', ['\"Respect Yourself\" is the name of a classic soul song by American R&B/gospel group The Staple Singers.', ' Released in late 1971 from their album \"\", the song became a crossover hit.', \" The Staple Singers' version peaked at #1 on KHJ, #12 on the Hot 100, #2 on the Hot Soul Singles chart and is one of the group's most recognizable hits.\", ' Bruce Willis recorded a cover version of the song in 1987 featuring The Pointer Sisters; it peaked at #5 on the Hot 100 and peaked at #7 in the UK.', ' In 2002 the song was inducted into the Grammy Hall of Fame, and in 2010 it was ranked #468 on the \"Rolling Stone\" list of the 500 Greatest Songs of All Time.']], [\"The J.B.'s\", [\"The J.B.'s (sometimes punctuated The JB's or The J.B.s) was the name of James Brown's band from 1970 through the early 1980s.\", ' On records the band was sometimes billed under alternate names such as The James Brown Soul Train, Maceo and the Macks, A.A.B.B., The First Family, and The Last Word.', \" In addition to backing Brown, the J.B.'s played behind Bobby Byrd, Lyn Collins, and other singers associated with the James Brown Revue, and performed and recorded as a self-contained group.\", ' In 2015, they were nominated for induction into the Rock and Roll Hall of Fame.']], ['Lost Someone', ['\"Lost Someone\" is a song recorded by James Brown in 1961.', ' It was written by Brown and Famous Flames members Bobby Byrd and Baby Lloyd Stallworth.', ' Like \"Please, Please, Please\" before it, the song\\'s lyrics combine a lament for lost love with a plea for forgiveness.', ' The single was a #2 R&B hit and reached #48 on the pop chart.', ' According to Brown, \"Lost Someone\" is based on the chord changes of the Conway Twitty song \"It\\'s Only Make Believe\".']], [\"Can't Count Me Out\", [\"Can't Count Me Out is the sixth studio album by American R&B/jazz singer Miki Howard, released in 1997 under Hush Records.\", ' Howard\\'s first studio recording in four years, the album contains a mixture of soulful ballads, some groovy R&B and a duet with father Clay Graham of gospel group, \"The Pilgrim Jubilees\".', ' Also featured are a few cover tunes, including Janis Ian\\'s \"At Seventeen\" produced by Robby Takac, member of the rock group Goo Goo Dolls, and Stevie Wonder\\'s \"I Love Every Little Thing About You\", a duet with pop/R&B singer Terence Trent D\\'Arby with Chaka Khan singing background vocals.']], ['Joya (singer)', ['Joya (birth name Joya Owens born 1973) in Detroit, Michigan, United States is an American R&B/soul singer - songwriter, who released her debut album \"Here I Am\" on Atlas/A&M record label.', \" Following her debut album release, Joya was a background session vocalist for various artists' including R&B/soul singer Mary J. Blige.\", ' She also toured as a background singer during Blige\\'s \"Share My World Tour\" in 1998.']], ['Valvin Roane', ['Valvin \"V\" Roane birth name given as Valvin Roane II, was born May 28, in Paulsboro, New Jersey is an American R&B/Soul singer and Songwriter who has set a solid foundation in the music industry.', ' Known in the music industry by his nickname \"V\", this talented vocalist has worked on projects with a host of renowned artists and producers in the music industry, including Anthony Hamilton, Jill Scott, Justin Timberlake, Musiq Soulchild and Will Smith to name a few.', ' His talents in singing, songwriting and producing has been compared to such talents as Stevie Wonder, Donnie Hathaway and Marvin Gaye.', ' His performances around the world with DJ Jazzy Jeff and as a background vocalist for Jill Scott (on her \"Buzz Tour\", \"Big Beautiful Tour\",\"The Real Thing Tour\" and \"The Light of The Sun Tour\" has received rousing ovations from audiences around the globe.', ' Known in many underground circles as thee pioneer of the philly \"neo-soul\" movement, \"V\" had many mind blowing records buzzing around the music industry for years before his 1st official debut album, \"The Revelation is Now Televised\" was released under BBE records in 2006, and features a spoken word piece from R&B/soul singer Jill Scott, \"Born Again\".', ' \"V\" aka V. Roane has continued to release great music which can be found on iTunes, Amazon and many more online digital music distributors.']], ['Solomon Burke', ['Solomon Burke (born James Solomon McDonald, March 21, 1940\\xa0– October 10, 2010) was an American preacher and singer, who shaped the sound of rhythm and blues as one of the founding fathers of soul music in the 1960s and a \"key transitional figure in the development of soul music from rhythm and blues.', ' He had a string of hits including \"Cry to Me\", \"If You Need Me\", \"Got to Get You Off My Mind\", \"Down in the Valley\" and \"Everybody Needs Somebody to Love\".', ' Burke was referred to as \"King Solomon\", the \"King of Rock \\'n\\' Soul\", \"Bishop of Soul\" and the \"Muhammad Ali of soul\".', ' Due to his minimal chart success in comparison to other soul music greats such as James Brown, Wilson Pickett and Otis Redding, Burke has been described as the genre\\'s \"most unfairly overlooked singer\" of its golden age.', ' Atlantic Records executive Jerry Wexler referred to Burke as \"the greatest male soul singer of all time\".']], ['Roy Brown (blues musician)', ['Roy James Brown (September 10, 1920 or 1925May 25, 1981) was an American R&B singer, songwriter and musician, who had a significant influence on the early development of rock and roll and the direction of R&B.', ' His original song and hit recording \"Good Rockin\\' Tonight\" has been covered by many artists including Wynonie Harris, Elvis Presley, Bruce Springsteen, Ricky Nelson, Jerry Lee Lewis, Pat Boone, James Brown, the Doors, and the rock group Montrose.', ' Brown was the first singer in recording history to sing R&B songs with a gospel-steeped delivery, which was then considered taboo by many churches.', ' In addition, his melismatic, pleading vocal style influenced notable artists such as B.B. King, Bobby Bland, Elvis Presley, Jackie Wilson, James Brown and Little Richard.']], ['Bobby Byrd', ['Bobby Howard Byrd (August 15, 1934 – September 12, 2007) was an American R&B/soul singer, songwriter, bandleader, talent scout, record producer, and musician, who played an integral and important part in the development of soul and funk music in association with James Brown.', \" Byrd began his career in 1952 as member of the gospel group the Gospel Starlighters, who later changed their name to the Avons in 1953 and the Five Royals in 1954, before settling with the name the Flames in 1955 prior to Brown's joining the group; their agent later changed it to The Famous Flames.\", ' Byrd was the actual founder of The Flames and is credited with the discovery of James Brown.', ' As group founder, and one of the longest-serving members of the group, Byrd was inducted into the Rock & Roll Hall of Fame posthumously in 2012.', \" Byrd was also a 1998 recipient of the Rhythm and Blues Foundation's Pioneer Award.\", ' Byrd helped to inspire the musical aspirations of James Brown, who launched his career with Byrd.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-09 18:10:18.813\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5adf21bc5542993344016bf4', 'answer': 'Savin Yeatman-Eiffel', 'question': 'Lost in Time includes the theme song for a French/Japanese anime series that was created by who?', 'supporting_facts': [['Lost in Time (Akino album)', 1], ['Ōban Star-Racers', 0]], 'context': [['Lost in Time (Akino album)', ['Lost in Time is the debut album of AKINO performing under the name \"AKINO from bless4\".', ' The album comprises all of the songs she performed for the anime series \"Genesis of Aquarion\" as well as the theme song for \"Ōban Star-Racers\".', ' It peaked at No. 25 on the Oricon Weekly Album Chart.']], ['List of Nichijou episodes', ['\"Nichijou\" is a 2011 Japanese anime television series produced by Kyoto Animation based on the comedy manga by Keiichi Arawi.', ' The series is directed by Tatsuya Ishihara and aired in Japan from April 3, 2011 to September 26, 2011.', ' It is also simulcasted by Crunchyroll under the name \"My Ordinary Life\".', ' Prior to the airing of the anime series, an original video animation episode, titled \"Nichijou Episode 0\", shipped with the sixth manga volume on March 12, 2011.', ' For the first 13 episodes in the first broadcast, the opening theme song is \"Hyadain no Kakakata Kataomoi - C\" (ヒャダインのカカカタ☆カタオモイ-C , lit.', ' \"Hyadain\\'s U-u-u-u☆unrequited Love - C\") by Hyadain, while the ending theme song is \"Zzz\" by Sayaka Sasaki.', ' For episodes 14 onwards, the opening theme is \"Hyadain no Jōjō Yūjō\" (ヒャダインのじょーじょーゆーじょー , Hyadain\\'s Amazing Friendship ) by Hyadain and its ending song varies every episode.']], [\"Lupin, l'incorreggibile Lupin\", ['\"Lupin, l\\'incorreggibile Lupin\" is a theme song written by Alessandra Valeri Manera and Ninni Carucci and sung by a fictional music group called \"Gli Amici di Lupin\" (The Lupin\\'s friends), which is composed by Enzo Draghi (vocals) and Simone D\\'Andrea (\"Sei furbo Lupin!\"', ' sentence).', ' It is the Italian theme song of \"Lupin the Third\" anime series used from 1987.', ' From 2004 the song is replaced by Giorgio Vanni\\'s \"Hallo Lupin\" in the first and second anime series, whereas third anime series keep \"Lupin, l\\'incorreggibile Lupin\" song.']], ['List of Ergo Proxy episodes', ['The following is a list of episodes for the Japanese anime series \"Ergo Proxy\".', ' \"Ergo Proxy\" was produced by Manglobe Inc. and directed by Shukou Murase.', ' It began broadcasting in Japan on February 25, 2006, on WOWOW.', ' The English version the anime is licensed by Geneon Entertainment.', ' \"Ergo Proxy\" was a DVD-only series, until syndication of the series on the Fuse channel began on June 9, 2007 at 12:30\\xa0a.m.', ' The series\\' opening theme song is \"Kiri\" by rock group Monoral, and is first shown in episode 3.', ' The ending theme song is \"Paranoid Android\", a well-known song by English alternative rock band Radiohead.', ' Another ending is on the Hulu version.']], ['List of Aikatsu Stars! episodes', ['\"Aikatsu Stars!', '\" is a Japanese anime television series produced by BN Pictures, and the successor to the original \"Aikatsu!', '\" anime series based on Bandai\\'s Data Carddass arcade machines.', ' The story follows a girl named Yume Nijino who enrolls at Yotsuboshi Gakuen (Four Star Academy) in order to become a top idol and join the popular group S4 which she admires.', ' The series began airing on TV Tokyo from April 7, 2016, succeeding the original \"Aikatsu!\"', ' anime series in its initial timeslot.', ' For the first 25 episodes, the opening theme is \"Start Line!\"', ' (スタートライン!', ' , Sutāto Rain! )', ' by Sena and Rie from AIKATSU☆STARS!', ', while the ending theme is \"episode Solo\" by Ruka, Nanase, Kana, and Miho from AIKATSU☆STARS!', '.', ' From episode 26 onwards, the opening theme is \"1, 2, Sing For you!\"', ' by Sena, Rie, Miki and Kana.', ' From episode 34 till episode 50 the opening theme is \"STAR JET!\"', ' (スタージェット！Sutā Jetto!)', ' by Sena, Rie, Kana and Miki.', ' The ending theme from episode 26 till episode 50 is \"So Beautiful Story\" by Ruka and Sena.']], ['List of Tytania episodes', ['The following is the list of episodes for the Japanese Anime series \"Tytania\".', ' The anime series is produced by Artland and sound productions by Magic Capsule.', ' The episodes are directed by Noboru Ishiguro based on the original novel created by Yoshiki Tanaka.', ' The characters from the anime are designed by Noboru Sugimitsu based on the original character designs by Haruhiko Mikimoto.', \" The series began airing on Japan's NHK broadcasting station on 9 October 2008.\", ' Two pieces of theme music are used, one opening and one ending theme.', ' The opening theme is titled \"Ano Sora wo, Ike\" performed by Ken Nishikiori while the ending theme is titled \"Lost in Space\" sung by Psychic Lover.']], ['List of One-Punch Man episodes', ['\"One-Punch Man\" is a Japanese anime series based on the webcomic created by One and its subsequent manga adaptation illustrated by Yusuke Murata.', ' Set in Z City, the story focuses on Saitama, a superhero who has grown bored as he has become so powerful that all of his battles end in a single punch.', ' The series was directed by Shingo Natsume at Madhouse and was written by Tomohiro Suzuki.', ' The series also features character design by Chikashi Kubota, who also served as chief animation director, and music by Makoto Miyazaki.', ' The series aired in Japan between October 5, 2015 and December 21, 2015 and was simulcast by Daisuki and Hulu.', ' The opening theme song is by JAM Project, and the ending theme is by Hiroko Moriguchi.', ' The ending theme for episode 12 is \"Kanashimi-tachi o Dakishimete\" (悲しみたちを抱きしめて , \"Hug Those Who Are Feeling Sad\" ) by Moriguchi.', ' An original video animation was released with the tenth manga volume on December 4, 2015.', ' Additional OVAs are included in Blu-ray Disc & DVD volumes of the series, which begin release from December 24, 2015.', ' The ending theme for the OVAs is \"Saitama no WanPan Ondo\" (サイタマのワンパン音頭 , \"Saitama\\'s One-Punch Song\" ) , performed by Makoto Furukawa.', ' The series is licensed in North America by Viz Media, who also streamed the series on its Neon Alley service, and in the United Kingdom by Kaze UK.', \" On television, the series began airing on Adult Swim's Toonami block on July 16, 2016.\", ' The first season of the anime received critical acclaim, praised for its animation quality, visuals and the overall story itself.']], ['Ōban Star-Racers', ['Ōban Star-Racers (オーバン・スターレーサーズ , Ōban Sutā Rēsāzu ) is a French/Japanese anime series created by Savin Yeatman-Eiffel of Sav!', ' The World Productions in association with multiple international companies.', ' Originally produced as a short movie titled Molly Star Racer, a television series was developed in cooperation with Jetix Europe, with animation production by HAL Film Maker and Pumpkin 3D, a large portion of which was done in Tokyo, Japan.', ' It aired in more than 100 countries including Japan.', ' In the US, the series aired on ABC Family and Jetix/Toon Disney between June and December 2006.']], ['List of Tantei Opera Milky Holmes episodes', ['Tantei Opera Milky Holmes is a Japanese anime series produced by J.C. Staff, based on the media franchise created by Bushiroad.', ' In a world where detectives and thieves battle against each other using supernatural abilities known as Toys, the story follows Sherlock Shellingford, Nero Yuzurizaki, Hercule Barton and Cordelia Glauca, a group of young detectives who suddenly lose their Toys and have to earn them back to avoid expulsion from Holmes Detective Academy.', ' The first series aired in Japan on Tokyo MX between October 7, 2010 and December 23, 2010, and was also streamed on Crunchyroll.', ' This was followed by a Summer Special episode which aired on August 26, 2011.', ' A second anime series, \"Tantei Opera Milky Holmes: Act 2\", aired in Japan between January 5, 2012 and March 22, 2012.', ' A second TV special aired on August 25, 2012.', ' For the first season, the opening theme is \"The Answer Is One!', ' Not!!\"', ' (正解はひとつ!じゃない!!', ' , Seikai wa Hitotsu!', ' Janai!! )', ' by Milky Holmes (Suzuko Mimori, Sora Tokui, Mikoi Sasaki and Izumi Kitta), whilst the ending theme is \"Instinct Doubt\" (本能のDOUBT , Honnou no DOUBT ) by Faylan.', ' The ending theme for the TV special is \"Party Party!\"', ' (パーティーパーティー!', ' , Pātī Pātī! )', ' by Milky Holmes.', ' For the second season, the opening theme is \"Mystery!', ' Mystery?', ' Happiness!!\"', ' (ナゾ!ナゾ?Happiness!!', ' , Nazo!', ' Nazo?', ' Happiness!! )', ' by Milky Holmes whilst the ending theme is \"Lovely Girls Anthem\" by Natsuko Aso.', ' The ending theme for the second TV special is \"A Place For You And Me\" (キミのなかのワタシ , Kimi no Naka no Watashi ) by Milky Holmes with SV Tribe.', ' A third season, \"Futari wa Milky Holmes\", aired between July 13, 2013 and September 28, 2013, focusing on a pair of detectives named Alice and Kazumi who face up against a thief unit known as Color the Phantom.', ' The opening theme is \"Glory Glowing☆Days\" (ぐろーりーぐろーいん☆DAYS , Gurōri Gurōin Deizu ) by Milky Holmes whilst the ending theme is \"Seishun Beginner!\"', ' (セイシュンビギナー!', ' , Seishun Biginā! )', ' by Ayasa Itō and Aimi Terakawa.', ' With the exception of \"Alternative\" and \"Futari wa\", each episode is named after a piece of famous detective fiction.']], ['List of Rage of Bahamut episodes', ['Rage of Bahamut (神撃のバハムート , Shingeki no Bahamut ) is a Japanese anime television series produced by MAPPA that is based on the \"Rage of Bahamut\" game.', ' The anime aired between October 6, 2014 and December 29, 2014 for 12 episodes.', \" On May 6, 2015, a second season was announced at the series' orchestra concert event.\", ' Titled Rage of Bahamut: Virgin Soul (神撃のバハムート VIRGIN SOUL , Shingeki no Bahamut: Virgin Soul ) , it premiered on April 7, 2017.', ' The opening theme song \"EXiSTENCE\" is performed by SiM, while the ending theme song \"Promised Land\" is performed by Risa Shimizu.', ' For season 2, the first opening theme song is \"LET iT END\" by SiM while the first ending theme song is \"Haikei Goodbye Sayonara\" (拝啓グッバイさようなら , lit.', ' Greetings, Goodbye, Farewell) by DAOKO.', ' The second opening theme is \"Walk This Way\" by THE BEAT GARDEN, while the second ending theme is \"Cinderella Step\" by DAOKO.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The initial node 'generate_answer' is missing required inputs: ['context', 'question']. You should provide these inputs by specifying the `inputs={'input_name': 'input_value'}` parameter in the `execute` method, or return the valid inputs in the `collate_func` when using `Evaluator`.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "logger.info(\"Evaluating workflow on test set...\")\n",
    "with suppress_logger_info():\n",
    "    results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "logger.info(f\"Evaluation metrics (before optimization): {results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "1726fb5f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "e9fa0aec",
   "metadata": {},
   "outputs": [],
   "source": [
    "# qa_workflow = {\n",
    "#     \"goal\": \"Provide a direct answer to the question based on the context, without including explanations or reasoning.\",\n",
    "#     \"tasks\": [\n",
    "#         {\n",
    "#             \"name\": \"answer_generate\",\n",
    "#             \"description\": \"Generate a direct answer to the question based on the context.\",\n",
    "#             \"inputs\": [\n",
    "#                 {\"name\": \"problem\", \"type\": \"str\", \"required\": True, \"description\": \"The question to answer directly.\"}\n",
    "#             ],\n",
    "#             \"outputs\": [\n",
    "#                 {\"name\": \"answer\", \"type\": \"str\", \"required\": True, \"description\": \"The direct answer to the question.\"}\n",
    "#             ],\n",
    "#             \"prompt_template\": StringTemplate(instruction=\"Think step by step to answer the question. You should explain your thinking process in the 'thought' field, and provide the final answer in the 'answer' field.\\nFormat your output in xml format, such as <thought>xxx</thought> and <answer>xxx</answer>.\"),\n",
    "#             \"parse_mode\": \"str\"\n",
    "#         }\n",
    "#     ] \n",
    "# }\n",
    "\n",
    "#generated_workflow\n",
    "qa_workflow = {\n",
    "    \"goal\": \"Provide a concise answer to the question using relevant context. The answer must be straightforward and avoid unnecessary explanations.\",\n",
    "    \"tasks\": [\n",
    "        {\n",
    "            \"name\": \"generate_answer\",\n",
    "            \"description\": \"Extract and formulate an answer from the given context.\",\n",
    "            \"inputs\": [\n",
    "                {\"name\": \"question\", \"type\": \"str\", \"required\": True, \"description\": \"The question that needs to be answered.\"},\n",
    "                {\"name\": \"context\", \"type\": \"str\", \"required\": True, \"description\": \"The background information pertinent to the question.\"}\n",
    "            ],\n",
    "            \"outputs\": [\n",
    "                {\"name\": \"answer\", \"type\": \"str\", \"required\": True, \"description\": \"The direct answer to the question.\"}\n",
    "            ],\n",
    "            \"prompt_template\": StringTemplate(instruction=\"Use the context to determine the best answer to the question. Provide your final answer in a clear format, without extra commentary or reasoning.\"),\n",
    "            \"parse_mode\": \"xml\"\n",
    "        }\n",
    "    ]\n",
    "}\n",
    "\n",
    "def collate_func(example: dict) -> dict:\n",
    "    context = \"\"\n",
    "    problem = \"Question: {}\\n\\nAnswer:\".format(example[\"question_new\"])\n",
    "    return {\"question\": problem, 'context':context}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "7434d744",
   "metadata": {},
   "outputs": [],
   "source": [
    "api_key = \"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\"\n",
    "# Function to encode the image\n",
    "import os\n",
    "os.environ[\"OPENAI_API_KEY\"] = api_key\n",
    "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n",
    "\n",
    "\n",
    "llm_config = OpenAILLMConfig(model=\"gpt-4o-mini-2024-07-18\", openai_key=OPENAI_API_KEY, top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
    "llm = OpenAILLM(config=llm_config)\n",
    "executor_llm = OpenAILLM(config=llm_config)\n",
    "optimizer_llm = OpenAILLM(config=llm_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "ead42143",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 19:30:58.894\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/adamson_train.json ...\u001b[0m\n",
      "\u001b[32m2025-12-27 19:30:58.895\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/adamson_train.json ...\u001b[0m\n",
      "\u001b[32m2025-12-27 19:30:58.895\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/adamson_test.json ...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "benchmark = PertQA()\n",
    "workflow_graph = SequentialWorkFlowGraph.from_dict(qa_workflow)\n",
    "agent_manager = AgentManager()\n",
    "agent_manager.add_agents_from_workflow(workflow_graph, executor_llm.config)\n",
    "\n",
    "evaluator = Evaluator(\n",
    "    llm=executor_llm, \n",
    "    agent_manager=agent_manager, \n",
    "    collate_func=collate_func, \n",
    "    num_workers=20, \n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "textgrad_optimizer = TextGradOptimizer(\n",
    "    graph=workflow_graph, \n",
    "    optimize_mode=\"all\",\n",
    "    executor_llm=executor_llm, \n",
    "    optimizer_llm=optimizer_llm,\n",
    "    batch_size=3,\n",
    "    max_steps=20,\n",
    "    evaluator=evaluator,\n",
    "    eval_every_n_steps=1,\n",
    "    eval_rounds=1,\n",
    "    save_interval=None,\n",
    "    save_path=\"./\",\n",
    "    rollback=True,\n",
    "    constraints=[]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "1e8e1560",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# len(benchmark._fulldata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "5a9f95bc",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 19:30:59.879\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m1\u001b[0m - \u001b[1mEvaluating workflow on test set...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 6/2500 [00:00<04:35,  9.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 16/2500 [00:01<02:05, 19.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 23/2500 [00:01<02:16, 18.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 26/2500 [00:01<02:28, 16.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 34/2500 [00:01<01:31, 26.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 38/2500 [00:02<02:25, 16.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 46/2500 [00:02<01:59, 20.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 52/2500 [00:02<01:38, 24.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 56/2500 [00:03<01:56, 21.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 62/2500 [00:03<02:23, 16.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 69/2500 [00:03<01:55, 21.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 74/2500 [00:04<02:19, 17.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 77/2500 [00:04<02:26, 16.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 86/2500 [00:04<01:57, 20.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 93/2500 [00:05<01:47, 22.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 99/2500 [00:05<02:02, 19.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 103/2500 [00:05<01:42, 23.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 109/2500 [00:06<02:44, 14.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 118/2500 [00:06<02:07, 18.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 124/2500 [00:07<02:12, 17.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 127/2500 [00:07<02:13, 17.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 133/2500 [00:07<02:09, 18.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 136/2500 [00:07<02:14, 17.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 143/2500 [00:08<01:58, 19.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 146/2500 [00:08<01:50, 21.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 153/2500 [00:08<01:46, 21.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 159/2500 [00:08<01:54, 20.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 165/2500 [00:09<02:11, 17.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 169/2500 [00:09<01:52, 20.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 175/2500 [00:09<01:34, 24.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 182/2500 [00:10<02:08, 17.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 191/2500 [00:10<01:33, 24.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 194/2500 [00:10<01:44, 22.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 197/2500 [00:10<02:12, 17.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 205/2500 [00:11<01:54, 20.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 208/2500 [00:11<02:11, 17.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 211/2500 [00:11<02:02, 18.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▊         | 216/2500 [00:12<02:18, 16.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 219/2500 [00:12<02:06, 17.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 223/2500 [00:12<02:32, 14.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 232/2500 [00:12<01:48, 20.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 238/2500 [00:13<01:52, 20.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 247/2500 [00:13<02:00, 18.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 254/2500 [00:14<01:39, 22.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 257/2500 [00:14<02:18, 16.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 265/2500 [00:14<01:50, 20.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 270/2500 [00:14<01:58, 18.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 276/2500 [00:15<02:07, 17.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 281/2500 [00:15<02:02, 18.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 287/2500 [00:15<01:45, 21.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 293/2500 [00:16<01:56, 19.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 297/2500 [00:16<01:40, 22.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 302/2500 [00:16<01:57, 18.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 308/2500 [00:17<02:04, 17.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 311/2500 [00:17<02:02, 17.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 319/2500 [00:17<01:30, 24.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 322/2500 [00:17<01:46, 20.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 329/2500 [00:18<01:46, 20.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 338/2500 [00:18<01:44, 20.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 345/2500 [00:18<01:32, 23.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 348/2500 [00:18<01:29, 24.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 351/2500 [00:19<02:09, 16.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 354/2500 [00:19<02:05, 17.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 366/2500 [00:19<01:38, 21.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 369/2500 [00:20<01:45, 20.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 376/2500 [00:20<01:42, 20.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 382/2500 [00:20<01:39, 21.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 386/2500 [00:20<01:25, 24.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 392/2500 [00:21<01:46, 19.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 396/2500 [00:21<01:31, 23.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 399/2500 [00:21<01:42, 20.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 404/2500 [00:21<02:20, 14.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 410/2500 [00:22<01:33, 22.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 418/2500 [00:22<01:34, 22.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 427/2500 [00:22<01:29, 23.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 430/2500 [00:23<01:53, 18.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 436/2500 [00:23<01:40, 20.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 444/2500 [00:23<01:18, 26.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 447/2500 [00:23<01:46, 19.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 450/2500 [00:24<02:07, 16.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 460/2500 [00:24<01:25, 23.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 463/2500 [00:24<01:36, 21.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 469/2500 [00:25<01:43, 19.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 473/2500 [00:25<01:36, 21.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 476/2500 [00:25<01:41, 19.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 484/2500 [00:25<01:30, 22.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 490/2500 [00:25<01:29, 22.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 494/2500 [00:26<01:38, 20.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 497/2500 [00:26<01:46, 18.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 503/2500 [00:26<01:45, 18.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 509/2500 [00:26<01:30, 22.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 513/2500 [00:27<01:28, 22.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 519/2500 [00:27<01:41, 19.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 522/2500 [00:27<01:38, 20.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 529/2500 [00:27<01:25, 23.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 532/2500 [00:28<01:43, 18.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 536/2500 [00:28<01:32, 21.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 542/2500 [00:28<01:33, 20.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 546/2500 [00:28<01:32, 21.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 552/2500 [00:29<01:29, 21.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 555/2500 [00:29<01:44, 18.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 558/2500 [00:29<01:33, 20.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 561/2500 [00:29<01:38, 19.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 568/2500 [00:29<01:39, 19.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 574/2500 [00:30<01:29, 21.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 577/2500 [00:30<01:47, 17.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 583/2500 [00:30<01:34, 20.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 586/2500 [00:30<01:33, 20.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 592/2500 [00:31<01:37, 19.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 596/2500 [00:31<01:22, 23.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 606/2500 [00:31<01:20, 23.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 610/2500 [00:31<01:16, 24.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 613/2500 [00:32<01:27, 21.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 616/2500 [00:32<01:53, 16.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 624/2500 [00:32<01:17, 24.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 629/2500 [00:32<01:19, 23.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 632/2500 [00:33<01:30, 20.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 638/2500 [00:33<01:38, 18.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 647/2500 [00:33<01:12, 25.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 650/2500 [00:33<01:37, 19.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 656/2500 [00:34<01:37, 18.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 662/2500 [00:34<01:12, 25.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 665/2500 [00:34<01:12, 25.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 672/2500 [00:34<01:27, 20.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 675/2500 [00:35<01:28, 20.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 683/2500 [00:35<01:28, 20.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 686/2500 [00:35<01:22, 21.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 690/2500 [00:35<01:20, 22.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 697/2500 [00:36<01:27, 20.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 700/2500 [00:36<01:35, 18.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 707/2500 [00:36<01:21, 22.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 710/2500 [00:36<01:28, 20.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 717/2500 [00:37<01:13, 24.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 721/2500 [00:37<01:16, 23.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 724/2500 [00:37<01:48, 16.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 727/2500 [00:37<01:55, 15.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 735/2500 [00:38<01:19, 22.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 742/2500 [00:38<01:19, 22.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 745/2500 [00:38<01:24, 20.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 748/2500 [00:38<01:43, 16.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 758/2500 [00:39<01:05, 26.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 762/2500 [00:39<01:18, 22.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 765/2500 [00:39<01:22, 21.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 772/2500 [00:39<01:30, 19.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 775/2500 [00:40<01:22, 20.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 781/2500 [00:40<01:30, 18.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 787/2500 [00:40<01:27, 19.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 793/2500 [00:41<01:26, 19.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 796/2500 [00:41<01:32, 18.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 801/2500 [00:41<01:37, 17.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 805/2500 [00:41<01:24, 20.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 811/2500 [00:42<01:24, 20.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 814/2500 [00:42<01:31, 18.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 820/2500 [00:42<01:31, 18.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 826/2500 [00:42<01:23, 19.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 829/2500 [00:42<01:18, 21.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 834/2500 [00:43<01:33, 17.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 839/2500 [00:43<01:31, 18.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 844/2500 [00:43<01:06, 24.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 851/2500 [00:44<01:29, 18.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 855/2500 [00:44<01:19, 20.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 861/2500 [00:44<01:18, 20.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 867/2500 [00:44<01:21, 20.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 871/2500 [00:45<01:25, 19.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 875/2500 [00:45<01:13, 22.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 885/2500 [00:45<01:08, 23.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 888/2500 [00:45<01:21, 19.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 891/2500 [00:46<01:29, 17.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 897/2500 [00:46<01:03, 25.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 904/2500 [00:46<01:11, 22.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 907/2500 [00:46<01:35, 16.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 917/2500 [00:47<01:06, 23.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 921/2500 [00:47<01:14, 21.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 924/2500 [00:47<01:28, 17.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 931/2500 [00:47<01:21, 19.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 937/2500 [00:48<01:21, 19.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 940/2500 [00:48<01:17, 20.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 948/2500 [00:48<01:25, 18.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 954/2500 [00:49<01:14, 20.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 957/2500 [00:49<01:20, 19.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 960/2500 [00:49<01:25, 18.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 966/2500 [00:49<01:23, 18.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 972/2500 [00:50<01:07, 22.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 976/2500 [00:50<00:57, 26.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 979/2500 [00:50<01:11, 21.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 986/2500 [00:50<01:11, 21.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 993/2500 [00:50<01:02, 23.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 996/2500 [00:51<01:26, 17.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 999/2500 [00:51<01:25, 17.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1006/2500 [00:51<01:12, 20.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1013/2500 [00:51<01:03, 23.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1016/2500 [00:52<01:18, 18.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1020/2500 [00:52<01:10, 21.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1029/2500 [00:52<01:01, 23.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1032/2500 [00:52<01:14, 19.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1036/2500 [00:53<01:06, 22.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1042/2500 [00:53<01:14, 19.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1047/2500 [00:53<01:18, 18.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1050/2500 [00:53<01:15, 19.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1055/2500 [00:54<01:33, 15.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1059/2500 [00:54<01:13, 19.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1062/2500 [00:54<01:17, 18.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1064/2500 [00:54<01:48, 13.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1071/2500 [00:55<01:13, 19.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1077/2500 [00:55<01:06, 21.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1081/2500 [00:55<00:58, 24.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1087/2500 [00:55<01:16, 18.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1094/2500 [00:56<01:04, 21.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1097/2500 [00:56<01:03, 22.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1100/2500 [00:56<01:17, 17.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1104/2500 [00:56<01:05, 21.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1112/2500 [00:57<00:56, 24.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1119/2500 [00:57<00:56, 24.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1125/2500 [00:57<01:15, 18.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1128/2500 [00:58<01:22, 16.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1136/2500 [00:58<00:50, 27.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1140/2500 [00:58<00:56, 23.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1143/2500 [00:58<01:12, 18.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1151/2500 [00:58<00:57, 23.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1154/2500 [00:59<00:56, 23.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 1160/2500 [00:59<01:14, 17.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1168/2500 [00:59<00:58, 22.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1172/2500 [00:59<00:50, 26.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1175/2500 [01:00<00:59, 22.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1181/2500 [01:00<01:15, 17.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1190/2500 [01:00<00:50, 26.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1193/2500 [01:01<01:10, 18.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1199/2500 [01:01<01:12, 18.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1204/2500 [01:01<00:54, 23.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1210/2500 [01:01<01:02, 20.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▊     | 1218/2500 [01:02<00:58, 21.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1222/2500 [01:02<00:52, 24.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1225/2500 [01:02<01:09, 18.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1231/2500 [01:02<01:09, 18.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1239/2500 [01:03<00:45, 27.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1243/2500 [01:03<01:02, 20.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1246/2500 [01:03<01:08, 18.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1252/2500 [01:03<01:02, 20.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1255/2500 [01:04<01:39, 12.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1261/2500 [01:05<02:10,  9.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1274/2500 [01:05<01:15, 16.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1281/2500 [01:06<01:04, 18.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 1284/2500 [01:06<01:06, 18.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1291/2500 [01:06<01:02, 19.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1294/2500 [01:06<01:16, 15.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1300/2500 [01:07<00:52, 22.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1304/2500 [01:07<00:53, 22.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1310/2500 [01:07<00:59, 19.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1313/2500 [01:07<01:07, 17.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1316/2500 [01:07<01:02, 19.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1322/2500 [01:08<01:01, 19.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1329/2500 [01:08<00:56, 20.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1332/2500 [01:08<00:59, 19.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▎    | 1339/2500 [01:09<00:57, 20.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1342/2500 [01:09<00:56, 20.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1348/2500 [01:09<00:58, 19.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1351/2500 [01:09<00:55, 20.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1356/2500 [01:10<01:03, 18.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1359/2500 [01:10<01:03, 18.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1364/2500 [01:10<01:02, 18.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1368/2500 [01:10<01:02, 18.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1374/2500 [01:11<01:00, 18.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1381/2500 [01:11<00:51, 21.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1384/2500 [01:11<00:54, 20.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1387/2500 [01:11<01:08, 16.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1392/2500 [01:12<01:11, 15.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1400/2500 [01:12<00:51, 21.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1403/2500 [01:12<01:00, 18.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1409/2500 [01:12<00:51, 21.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1415/2500 [01:13<00:55, 19.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1421/2500 [01:13<00:53, 20.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1424/2500 [01:13<00:55, 19.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1428/2500 [01:13<00:56, 18.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1433/2500 [01:14<00:58, 18.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1438/2500 [01:14<00:51, 20.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1441/2500 [01:14<00:58, 17.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1446/2500 [01:15<01:01, 17.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1451/2500 [01:15<00:58, 17.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1459/2500 [01:15<00:45, 23.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1462/2500 [01:15<00:48, 21.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▊    | 1468/2500 [01:16<00:52, 19.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1471/2500 [01:16<00:58, 17.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1473/2500 [01:16<01:03, 16.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1481/2500 [01:16<00:39, 25.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1487/2500 [01:17<00:46, 21.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1490/2500 [01:17<00:49, 20.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1498/2500 [01:17<00:50, 19.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1501/2500 [01:17<00:54, 18.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1510/2500 [01:18<00:37, 26.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1514/2500 [01:18<00:52, 18.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1518/2500 [01:18<00:49, 19.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1524/2500 [01:18<00:45, 21.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1527/2500 [01:19<00:48, 20.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1530/2500 [01:19<00:52, 18.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 1535/2500 [01:19<00:53, 18.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1540/2500 [01:19<00:39, 24.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1543/2500 [01:20<00:56, 16.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1551/2500 [01:20<00:43, 21.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1559/2500 [01:20<00:37, 24.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1562/2500 [01:20<00:44, 20.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1571/2500 [01:21<00:42, 21.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1576/2500 [01:21<00:34, 26.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1580/2500 [01:21<00:55, 16.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1589/2500 [01:22<00:40, 22.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1595/2500 [01:22<00:41, 22.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1598/2500 [01:22<00:50, 18.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1601/2500 [01:22<00:54, 16.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1608/2500 [01:23<00:42, 20.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1612/2500 [01:23<00:38, 23.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1619/2500 [01:23<00:35, 25.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1627/2500 [01:23<00:43, 20.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1630/2500 [01:24<00:40, 21.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1636/2500 [01:24<00:39, 21.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1639/2500 [01:24<00:41, 20.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1645/2500 [01:24<00:40, 20.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1653/2500 [01:25<00:43, 19.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1656/2500 [01:25<00:47, 17.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1660/2500 [01:25<00:46, 18.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1666/2500 [01:26<00:40, 20.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1670/2500 [01:26<00:45, 18.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1673/2500 [01:26<00:40, 20.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1679/2500 [01:26<00:49, 16.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1686/2500 [01:27<00:37, 21.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1691/2500 [01:27<00:31, 25.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1694/2500 [01:27<00:37, 21.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1702/2500 [01:27<00:39, 20.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1708/2500 [01:28<00:40, 19.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1711/2500 [01:28<00:40, 19.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 1714/2500 [01:28<00:42, 18.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 1718/2500 [01:28<00:55, 14.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 1723/2500 [01:29<00:57, 13.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1731/2500 [01:29<00:29, 25.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1735/2500 [01:29<00:33, 22.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 1741/2500 [01:30<00:47, 15.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 1745/2500 [01:30<00:43, 17.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 1753/2500 [01:30<00:36, 20.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1756/2500 [01:30<00:40, 18.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1759/2500 [01:31<00:42, 17.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1766/2500 [01:31<00:39, 18.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1774/2500 [01:31<00:29, 24.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1781/2500 [01:32<00:35, 20.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 1785/2500 [01:32<00:31, 22.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1788/2500 [01:32<00:39, 17.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1792/2500 [01:32<00:35, 20.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1795/2500 [01:32<00:41, 17.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1802/2500 [01:33<00:35, 19.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1808/2500 [01:33<00:32, 20.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1812/2500 [01:33<00:27, 25.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1815/2500 [01:34<00:42, 16.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1821/2500 [01:34<00:37, 17.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1827/2500 [01:34<00:27, 24.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1830/2500 [01:34<00:35, 19.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1833/2500 [01:34<00:37, 17.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▎  | 1840/2500 [01:35<00:29, 22.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1845/2500 [01:35<00:29, 21.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1848/2500 [01:35<00:37, 17.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1856/2500 [01:35<00:29, 21.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1860/2500 [01:36<00:25, 24.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 1863/2500 [01:36<00:33, 19.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 1871/2500 [01:36<00:29, 21.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 1874/2500 [01:36<00:27, 22.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 1880/2500 [01:37<00:33, 18.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1883/2500 [01:37<00:31, 19.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1889/2500 [01:37<00:32, 18.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1892/2500 [01:37<00:32, 18.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1895/2500 [01:37<00:29, 20.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1902/2500 [01:38<00:30, 19.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1905/2500 [01:38<00:33, 17.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▋  | 1910/2500 [01:38<00:32, 18.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1913/2500 [01:38<00:28, 20.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1920/2500 [01:39<00:28, 20.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1924/2500 [01:39<00:24, 23.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1927/2500 [01:39<00:29, 19.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1933/2500 [01:40<00:34, 16.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1939/2500 [01:40<00:24, 22.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1945/2500 [01:40<00:30, 18.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1948/2500 [01:40<00:28, 19.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1955/2500 [01:41<00:26, 20.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1961/2500 [01:41<00:26, 20.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 1964/2500 [01:41<00:26, 19.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1971/2500 [01:41<00:23, 22.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1974/2500 [01:42<00:32, 16.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1978/2500 [01:42<00:33, 15.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1982/2500 [01:42<00:29, 17.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 1988/2500 [01:42<00:24, 20.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 1995/2500 [01:43<00:25, 19.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 1999/2500 [01:43<00:22, 21.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2006/2500 [01:43<00:25, 19.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2009/2500 [01:44<00:27, 17.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2017/2500 [01:44<00:21, 22.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2020/2500 [01:44<00:24, 19.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2027/2500 [01:44<00:23, 20.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2030/2500 [01:45<00:21, 22.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2033/2500 [01:45<00:24, 18.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2040/2500 [01:45<00:23, 19.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2047/2500 [01:45<00:20, 22.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2054/2500 [01:46<00:18, 24.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2057/2500 [01:46<00:23, 18.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2065/2500 [01:46<00:16, 26.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2069/2500 [01:47<00:23, 18.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2078/2500 [01:47<00:21, 19.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2081/2500 [01:47<00:20, 20.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2088/2500 [01:47<00:18, 21.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2091/2500 [01:48<00:21, 19.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2097/2500 [01:48<00:20, 20.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2104/2500 [01:48<00:16, 23.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2107/2500 [01:48<00:21, 18.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2114/2500 [01:49<00:17, 22.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2118/2500 [01:49<00:15, 24.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2125/2500 [01:49<00:19, 19.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2134/2500 [01:50<00:13, 26.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2141/2500 [01:50<00:19, 18.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2147/2500 [01:50<00:18, 19.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2150/2500 [01:51<00:18, 18.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2155/2500 [01:51<00:14, 23.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2158/2500 [01:51<00:19, 17.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2165/2500 [01:51<00:15, 21.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2173/2500 [01:51<00:11, 27.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2180/2500 [01:52<00:17, 18.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2183/2500 [01:52<00:15, 20.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2191/2500 [01:52<00:12, 24.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2194/2500 [01:53<00:16, 18.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2197/2500 [01:53<00:16, 18.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2203/2500 [01:53<00:16, 18.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2207/2500 [01:53<00:14, 19.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 2213/2500 [01:54<00:13, 21.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2216/2500 [01:54<00:14, 19.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2223/2500 [01:54<00:14, 19.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2226/2500 [01:54<00:12, 21.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2232/2500 [01:55<00:14, 18.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2235/2500 [01:55<00:13, 20.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2242/2500 [01:55<00:13, 19.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2247/2500 [01:55<00:10, 23.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2250/2500 [01:56<00:12, 20.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2253/2500 [01:56<00:13, 17.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2261/2500 [01:56<00:10, 23.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2264/2500 [01:56<00:10, 21.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2267/2500 [01:57<00:15, 15.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2275/2500 [01:57<00:11, 20.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2279/2500 [01:57<00:10, 21.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2286/2500 [01:57<00:10, 21.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2289/2500 [01:57<00:10, 19.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2298/2500 [01:58<00:07, 26.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2305/2500 [01:58<00:09, 20.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2310/2500 [01:58<00:07, 25.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2316/2500 [01:59<00:08, 22.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2323/2500 [01:59<00:08, 20.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2326/2500 [01:59<00:07, 21.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2332/2500 [01:59<00:08, 20.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2337/2500 [02:00<00:09, 18.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2341/2500 [02:00<00:07, 22.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2348/2500 [02:00<00:07, 20.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2351/2500 [02:00<00:07, 21.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2357/2500 [02:01<00:06, 21.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2361/2500 [02:01<00:05, 25.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2364/2500 [02:01<00:07, 19.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2372/2500 [02:01<00:06, 19.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2379/2500 [02:02<00:04, 25.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2382/2500 [02:02<00:05, 21.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2385/2500 [02:02<00:05, 20.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2391/2500 [02:02<00:05, 20.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2394/2500 [02:02<00:05, 21.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2400/2500 [02:03<00:04, 21.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2403/2500 [02:03<00:04, 19.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▋| 2411/2500 [02:03<00:04, 20.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2418/2500 [02:04<00:04, 19.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2424/2500 [02:04<00:04, 17.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2428/2500 [02:04<00:03, 21.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2434/2500 [02:05<00:03, 17.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2437/2500 [02:05<00:03, 18.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2443/2500 [02:05<00:03, 18.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2446/2500 [02:05<00:02, 18.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2448/2500 [02:06<00:05,  9.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▊| 2463/2500 [02:07<00:02, 13.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2466/2500 [02:07<00:02, 13.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2474/2500 [02:07<00:01, 18.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2480/2500 [02:08<00:01, 17.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2483/2500 [02:08<00:01, 15.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2494/2500 [02:08<00:00, 19.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2497/2500 [02:09<00:00, 17.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 2500/2500 [02:09<00:00, 19.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2025-12-27 19:33:09.396\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m4\u001b[0m - \u001b[1mEvaluation metrics (before optimization): {'f1': 0.0, 'em': 0.0, 'acc': 0.8336}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "logger.info(\"Evaluating workflow on test set...\")\n",
    "with suppress_logger_info():\n",
    "    results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "logger.info(f\"Evaluation metrics (before optimization): {results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "f3cdb546",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'f1': 0.0, 'em': 0.0, 'acc': 0.8336}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7132c3af",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3b6485e4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-19 10:40:44.863\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m1\u001b[0m - \u001b[1mEvaluating workflow on test set...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 1/500 [00:01<13:36,  1.64s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|▏         | 7/500 [00:02<01:44,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 12/500 [00:02<01:05,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 14/500 [00:02<01:10,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 16/500 [00:03<01:10,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 18/500 [00:03<01:11,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 19/500 [00:03<01:24,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 23/500 [00:04<01:09,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 27/500 [00:04<01:02,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 29/500 [00:04<00:55,  8.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.42857142857142855, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 30/500 [00:05<00:59,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 34/500 [00:05<00:55,  8.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 36/500 [00:05<00:56,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 38/500 [00:06<00:53,  8.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7272727272727273, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.43750000000000006, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 41/500 [00:06<00:53,  8.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcf78190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcf740c0>, 171997.430313798)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcf78510>\n",
      "Evaluating workflow:   8%|▊         | 42/500 [00:07<01:42,  4.45it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcf326d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcfab770>, 171997.84255116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcf324d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcda5950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdce69e10>, 171999.635562925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdccbd350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdce20190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcf47c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdce3bc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcf91e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcdabd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdccbaf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdce14710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcf773f0>, 171998.727051963)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcf47210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcf77a10>, 171998.977938194)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdce3bb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcf770e0>, 171999.380616192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdce20510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdd09ab30>, 171999.045251794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcdabdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdce684b0>, 171999.711702994)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcf92150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdce69940>, 171999.778501284)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdccbb490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcf76e40>, 171998.759199915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdce14dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdccebe50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdce6a190>, 172000.368671392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcceb490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcd8db50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcc30fa0>, 172001.826479998)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcc52690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdccbbd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcd6c310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcd88d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcd98110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcd99e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcbbcbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcd311d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcd49010>, 172000.522768615)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcd89450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdd0c4600>, 172000.263724429)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcd9a050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdd0c6d60>, 172000.290404383)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcbbca10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcd49a20>, 172001.223776257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcd98610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcf76c10>, 172001.728962247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdccbb450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcd48a60>, 171999.901801824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdce74a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcfab230>, 172000.897336038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcd31690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcbee4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcc33230>, 172001.864045037)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcbee710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcbda910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcb1c4b0>, 172001.580242823)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcbdb390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcb19610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcb1f070>, 172003.340882417)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcb1a450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcb73690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcb1f540>, 172002.936770117)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcb726d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcb54a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcbd94d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcac51d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcd89690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcaf7450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdd0c71c0>, 172002.070252372)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcac5090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdd0989f0>, 172001.816822232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcaf7790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcd48ad0>, 172002.033930614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afa0583ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcc32f90>, 172001.870469484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcd899d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcc30360>, 172003.362379494)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcb555d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcb72890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdca59a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcaadcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdca6d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdca2e050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcb1cb40>, 172003.635479094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdca2de50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdca1db00>, 172003.206860234)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcaac5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdd0c70e0>, 172003.519181605)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdca6d750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdca1d940>, 172004.05882007)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdca5a110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdca1c9f0>, 172002.917692199)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdca03890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdd252b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcf7b8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcf0ebd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcdbab10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcddca50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcdab510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcf33490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdce77690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcca4210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdccdda50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcd06a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcd14490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcd05910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcd2b250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcc26390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcc381d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcc69c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcc91790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcc048d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcc1d5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcaadd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcc6aad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcb69450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcbd92d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdc9d66d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdc9f1890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdcb82610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14afdca19f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdd1c94e0>, 172001.627136849)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdd252c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcf743d0>, 171998.561764952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcf7b650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcf756a0>, 171998.353670354)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcf0ed10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcf75b00>, 171999.129553291)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcf2d190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcf75d30>, 171998.292674677)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcddd190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcfaa890>, 171998.390622507)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcda8e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdd09b310>, 172002.347141219)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcf325d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcfa9400>, 171999.907417933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdce75e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcf766d0>, 171999.783460478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcca48d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdce69fd0>, 171999.735276245)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdccdd850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdce6b150>, 172000.441577989)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcd069d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdce6b770>, 171999.943483808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcd07c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcf76900>, 172000.572328971)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdccfee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdce6baf0>, 172000.082289221)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcd2a990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdce6a740>, 172001.666277891)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcc25190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcc30830>, 172001.318181138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcc38310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcc31160>, 172001.33165781)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcc6a290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcd49400>, 172001.705264966)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcc91c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdce6a970>, 172001.070853011)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcc04d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdd09b070>, 172001.701150373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcc1d410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcc32120>, 172001.823634248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcaadc10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcb1c130>, 172003.488025706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcc6a910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcb1d400>, 172003.269728343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcb69250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcd4b3f0>, 172003.717234793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcbda750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcb1f230>, 172003.192904264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdc9d5190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdcb1faf0>, 172003.081820228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdc9f1690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdce69be0>, 172002.322214003)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdcb82250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14afdca1c830>, 172003.077555978)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14afdca1a290>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 45/500 [00:07<01:31,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 50/500 [00:08<00:45,  9.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.13333333333333333, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 53/500 [00:08<00:34, 12.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 58/500 [00:08<00:38, 11.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 60/500 [00:09<00:58,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 62/500 [00:09<01:08,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 64/500 [00:09<01:08,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.125, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 68/500 [00:10<00:42, 10.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 70/500 [00:10<00:44,  9.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 72/500 [00:10<00:48,  8.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 74/500 [00:10<00:40, 10.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.30769230769230765, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 76/500 [00:11<00:48,  8.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 78/500 [00:11<00:48,  8.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 80/500 [00:12<01:27,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.16666666666666669, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 81/500 [00:12<01:20,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 85/500 [00:12<00:58,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 87/500 [00:12<00:55,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 88/500 [00:12<00:54,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 93/500 [00:13<00:35, 11.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 95/500 [00:13<00:36, 11.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 98/500 [00:14<01:03,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 100/500 [00:14<01:19,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 103/500 [00:15<01:01,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 107/500 [00:15<00:38, 10.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3636363636363636, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 109/500 [00:15<00:35, 11.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 111/500 [00:16<01:01,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.9090909090909091, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 113/500 [00:16<00:57,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 114/500 [00:16<01:02,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4799999999999999, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 115/500 [00:17<01:14,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 119/500 [00:17<00:49,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.03333333333333333, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 123/500 [00:17<00:36, 10.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.27272727272727276, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 125/500 [00:18<00:45,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.7368421052631579, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 127/500 [00:18<00:41,  9.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 129/500 [00:18<00:55,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 133/500 [00:19<00:44,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 137/500 [00:19<00:33, 10.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 141/500 [00:20<00:48,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 143/500 [00:20<00:39,  8.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 145/500 [00:20<01:01,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 151/500 [00:21<00:35,  9.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 155/500 [00:21<00:37,  9.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 157/500 [00:22<00:46,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 159/500 [00:22<01:07,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 161/500 [00:23<01:07,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 165/500 [00:23<00:50,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.7692307692307693, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 170/500 [00:24<00:32, 10.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.7142857142857143, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 172/500 [00:24<00:38,  8.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 176/500 [00:25<01:08,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 177/500 [00:26<01:08,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 181/500 [00:26<00:52,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 183/500 [00:26<00:41,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5333333333333333, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 187/500 [00:27<00:36,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 189/500 [00:27<00:38,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 191/500 [00:27<00:45,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 194/500 [00:28<00:32,  9.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 196/500 [00:28<01:00,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 198/500 [00:29<00:57,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 200/500 [00:29<00:53,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.08333333333333334, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 202/500 [00:29<01:00,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.125, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 206/500 [00:30<00:48,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 207/500 [00:30<00:48,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7777777777777778, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 210/500 [00:30<00:35,  8.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 214/500 [00:31<00:32,  8.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 215/500 [00:31<00:35,  7.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 217/500 [00:31<00:40,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 218/500 [00:32<00:41,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 219/500 [00:32<01:14,  3.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 222/500 [00:33<00:55,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.21428571428571425, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 224/500 [00:33<00:40,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 227/500 [00:33<00:48,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 229/500 [00:34<00:51,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 232/500 [00:34<00:30,  8.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3636363636363636, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 235/500 [00:34<00:24, 11.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 237/500 [00:35<00:35,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 239/500 [00:35<00:29,  8.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 241/500 [00:35<00:29,  8.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 243/500 [00:35<00:33,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 246/500 [00:36<00:40,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 247/500 [00:36<00:45,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 249/500 [00:37<00:41,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 251/500 [00:37<00:39,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 252/500 [00:37<00:43,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 254/500 [00:37<00:37,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 258/500 [00:38<00:30,  8.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.11764705882352941, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 259/500 [00:38<00:35,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 261/500 [00:38<00:37,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 262/500 [00:38<00:33,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 266/500 [00:39<00:29,  7.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 268/500 [00:39<00:23,  9.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.22222222222222224, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 270/500 [00:39<00:26,  8.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 272/500 [00:40<00:39,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 275/500 [00:40<00:30,  7.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.039603960396039604, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 277/500 [00:40<00:30,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 281/500 [00:41<00:30,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 283/500 [00:41<00:27,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 285/500 [00:42<00:27,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 289/500 [00:42<00:25,  8.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.23529411764705882, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 292/500 [00:42<00:20, 10.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 294/500 [00:42<00:20,  9.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "\u001b[32m2025-12-19 10:41:27.955\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5a83d7535542993344746093', 'answer': 'Los Angeles Xtreme, San Francisco Demons and Memphis Maniax', 'question': 'What other teams played in the same division that Chad Clements played in?', 'supporting_facts': [['Chuck Clements', 4], ['Las Vegas Outlaws (XFL)', 0], ['Las Vegas Outlaws (XFL)', 1]], 'context': [['Bertram Clements', ['Bertram Arthur Clements (1 December 1913 – July 2000) was an English footballer who represented Great Britain at the 1936 Summer Olympics.', ' Clements played amateur football for Casuals.']], ['1999–2000 BAI Basket', ['The 1999–2000 Season of BAI Basket (31st edition) ran from November 20, 2008 through May 16, 2000, with 8 teams playing in three different stages: in stage one (regular season) teams played a double round robin system.', ' In stage two, the six best teams played a single round robin tournament in serie A and the last six did the same for the consolation group, serie B. Finally, in stage three (final four) the best four teams from serie A played in a round robin at four rounds for the title.', ' The winners of the regular season and of the serie A are awarded a bonus point for the serie A and the final four, respectively.']], ['2007–08 BAI Basket', ['The 2007-2008 Season of BAI Basket (30th edition) ran from November 21, 2008 through May 16, 2009, with 12 teams playing in three different stages: in stage one (regular season) teams played a double round robin system.', ' In stage two, the six best teams played a single round robin tournament in serie A and the last six did the same for the consolation group, serie B. Finally, in stage three (final four) the best four teams from serie A played in a round robin at four rounds for the title.', ' The winners of the regular season and of the serie A are awarded a bonus point for the serie A and the final four, respectively.']], ['2009–10 BAI Basket', ['The 2009-2010 Season of BAI Basket (32nd edition) ran from November 13, 2009 to June 15, 2010, with 12 teams playing in three different stages: in stage one (regular season) teams played a double round robin system.', ' In stage two, the six best teams played a single round robin tournament in serie A and the last six did the same for the consolation group, serie B. Finally, in stage three (final four) the best four teams from serie A played in a round robin at four rounds for the title.', ' The winners of the regular season and of the serie A are awarded a bonus point for the serie A and the final four, respectively.']], ['Las Vegas Outlaws (XFL)', ['The Las Vegas Outlaws were an American football team in the XFL.', ' They played in the Western Division with the Los Angeles Xtreme, San Francisco Demons and Memphis Maniax.', ' They played their home games at Sam Boyd Stadium.', ' The Outlaws hosted the first nationally televised XFL game on NBC against the New York/New Jersey Hitmen.']], ['Al-Minaa SC–Naft Al-Janoob SC rivalry', ['Southern Iraqi football clubs Al-Minaa and Naft Al-Janoob have been rivals since the 2004–05 season when Naft Al-Janoob club started playing in the Premier League.', ' The clubs are respectively from Al-Maqal and Al-Tamimia, in the same city Basra, and for this reason a match between the two teams is sometimes called a \"Basra Derby\".', ' Another name is often used in the press is \"South Derby\", which comes from the location of Basra province in southern Iraq.', ' The animosity intensified since the first match, as Naft Al-Janoob was not expected to win Al-Minaa 1–0, and the exaggerated protest by Al-Minaa supporters to referee of match Khalil Yousuf prompted him to retire arbitration forever.', ' and this animosity reached a peak during the 2010–11 season, when both teams played at the end of the season in the Premier League in a match, that if it end at a draw, Naft Al-Janoob will relegate to the Iraq Division One.', ' Indeed, the match ended in a draw, and Al-Minaa fans celebrated the relegation of Naft Al-Janoob, and considered it a winning of league title.', ' In the 2015–16 season, Naft Al-Janoob returned to avenge Al-Minaa, when both teams played at the end of the season in the Premier League.', ' Al-Minaa needed two goals to go to the final, but Naft Al-Janoob played a defensive squad until the end of the match, although they were losing 1–0.']], ['2008–09 BAI Basket', ['The 2008-2009 Season of BAI Basket (31st edition) ran from November 20, 2008 through May 16, 2009, with 12 teams playing in three different stages: in stage one (regular season) teams played a double round robin system.', ' In stage two, the six best teams played a single round robin tournament in serie A and the last six did the same for the consolation group, serie B. Finally, in stage three (final four) the best four teams from serie A played in a round robin at four rounds for the title.', ' The winners of the regular season and of the serie A are awarded a bonus point for the serie A and the final four, respectively.']], ['Newport News Dodgers', ['The Newport News Dodgers were a minor league baseball affiliate of the Brooklyn Dodgers between 1944 and 1955.', ' They played in the Piedmont League and were based in Newport News, Virginia.', ' Gil Hodges played for this team in 1946.', ' Previously, Newport News teams were the Newport News Builders (1942), Newport News Pilots (1941), Newport News Shipbuilders (1900-1901; 1911-1922).', ' The teams played at Peninsula War Memorial Stadium on Pembroke Avenue in Hampton, Virginia.', ' The stadium was build by Brooklyn Dodgers President Branch Rickey.', ' The Dodgers played there from 1948-1955.', \" Previously, Newport News teams played at Builders' Park on Warwick Road (1944-1947) and prior to that at a ballpark on Wickham Avenue on the East End of Newport News.\", \" The Dodgers' move to Los Angeles in 1955 caused the team to realign its minor league affiliations, ending Newport News' franchise.\"]], ['List of KHL vs NHL games', ['Although the NHL teams played against Soviet league teams during the Super Series between 1976 and 1991, there were no games between post-Soviet and NHL teams until 2008, when Metallurg Magnitogorsk played against the New York Rangers for the 2008 Victoria Cup.', ' Two years later, in 2010, marked the first time since 1990 that NHL teams played games on post-Soviet ice.']], ['Chuck Clements', ['Chad \"Chuck\" Clements (born September 29, 1973) is a former American football quarterback who played one season with the New York Jets of the National Football League (NFL).', ' He was drafted by the New York Jets in the sixth round of the 1997 NFL Draft.', ' He played college football at the University of Houston and attended Huntsville High School in Huntsville, Texas.', ' He was also a member of the Philadelphia Eagles, Denver Broncos, Berlin Thunder, Las Vegas Outlaws and Ottawa Renegades.', ' Clements was drafted fifth overall by the Las Vegas Outlaws in the 2001 XFL Draft but, because of a preseason injury, never played for them.']]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The input to LLMOutputParser.parse should be a str, but found <class 'NoneType'>.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 296/500 [00:43<00:18, 11.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 299/500 [00:43<00:32,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.21052631578947367, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 301/500 [00:44<00:29,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 303/500 [00:44<00:28,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 305/500 [00:44<00:32,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 308/500 [00:45<00:23,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 309/500 [00:45<00:24,  7.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4166666666666667, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 312/500 [00:45<00:20,  9.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 319/500 [00:46<00:19,  9.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.15384615384615385, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4375, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 321/500 [00:46<00:19,  9.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 323/500 [00:47<00:20,  8.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7272727272727273, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 325/500 [00:47<00:26,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 326/500 [00:47<00:25,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 329/500 [00:48<00:24,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 333/500 [00:48<00:25,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 336/500 [00:49<00:20,  7.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 338/500 [00:49<00:18,  8.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 341/500 [00:49<00:18,  8.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 343/500 [00:49<00:16,  9.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 345/500 [00:50<00:22,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2666666666666667, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 347/500 [00:50<00:26,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 350/500 [00:51<00:21,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 352/500 [00:51<00:22,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 355/500 [00:51<00:15,  9.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 357/500 [00:51<00:17,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 360/500 [00:52<00:18,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.09756097560975609, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 362/500 [00:52<00:17,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 363/500 [00:52<00:21,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 365/500 [00:53<00:18,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6363636363636364, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▎  | 368/500 [00:53<00:16,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 372/500 [00:53<00:18,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.05714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 375/500 [00:54<00:16,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 378/500 [00:54<00:19,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 380/500 [00:55<00:14,  8.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 383/500 [00:55<00:15,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 385/500 [00:55<00:14,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.13333333333333336, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.42105263157894735, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 388/500 [00:56<00:13,  8.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 390/500 [00:56<00:14,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 392/500 [00:56<00:17,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 397/500 [00:57<00:14,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3448275862068966, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 400/500 [00:57<00:10,  9.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 402/500 [00:57<00:11,  8.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 404/500 [00:58<00:11,  8.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 407/500 [00:58<00:11,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 408/500 [00:58<00:11,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 410/500 [00:59<00:12,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.24000000000000002, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 414/500 [00:59<00:09,  8.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4166666666666667, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 418/500 [00:59<00:09,  8.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 419/500 [01:00<00:10,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 422/500 [01:00<00:09,  8.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 424/500 [01:00<00:08,  9.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 429/500 [01:01<00:08,  8.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 432/500 [01:01<00:07,  9.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 434/500 [01:02<00:10,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 436/500 [01:02<00:10,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 440/500 [01:02<00:06,  8.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.38095238095238093, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 442/500 [01:03<00:06,  9.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 444/500 [01:03<00:09,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8235294117647058, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 446/500 [01:04<00:10,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7499999999999999, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 448/500 [01:04<00:09,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 451/500 [01:05<00:08,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5263157894736842, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 453/500 [01:05<00:07,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 458/500 [01:05<00:04,  9.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 460/500 [01:05<00:03, 10.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 467/500 [01:06<00:03,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 469/500 [01:07<00:05,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 472/500 [01:08<00:06,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 476/500 [01:08<00:03,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 478/500 [01:09<00:03,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 480/500 [01:09<00:03,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 483/500 [01:10<00:03,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.28571428571428575, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 485/500 [01:10<00:02,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.47619047619047616, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 488/500 [01:10<00:01,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 490/500 [01:11<00:01,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▊| 493/500 [01:11<00:01,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 494/500 [01:12<00:01,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 495/500 [01:12<00:01,  3.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 497/500 [01:13<00:00,  3.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 498/500 [01:14<00:00,  2.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.05, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 500/500 [01:14<00:00,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-19 10:41:59.740\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m4\u001b[0m - \u001b[1mEvaluation metrics (before optimization): {'f1': 0.6289114129091681, 'em': 0.46292585170340683, 'acc': 0.657314629258517}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "logger.info(\"Evaluating workflow on test set...\")\n",
    "with suppress_logger_info():\n",
    "    results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "logger.info(f\"Evaluation metrics (before optimization): {results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f9b6e042",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'f1': 0.6289114129091681, 'em': 0.46292585170340683, 'acc': 0.657314629258517}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "04e2e048",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "500"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(textgrad_optimizer.evaluator._evaluation_records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "eaa69e0d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "0.0\n",
      "1.0\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "accl = []\n",
    "for i in textgrad_optimizer.evaluator._evaluation_records.keys():\n",
    "    print(textgrad_optimizer.evaluator._evaluation_records[i]['metrics']['acc'])\n",
    "    accl .append(textgrad_optimizer.evaluator._evaluation_records[i]['metrics']['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4d10acf6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.718"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.mean(accl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "46d3dee7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#consider aflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c1bc0e23",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv(\"../../aflow_save.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d81e6d2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Unnamed: 0    247.500000\n",
       "f1              0.683997\n",
       "em              0.532258\n",
       "acc             0.627016\n",
       "dtype: float64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[0:496].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6d925c5b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "496"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1489//3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a36220ca",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}