{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b06fe5ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/PyPDF2/__init__.py:21: DeprecationWarning: PyPDF2 is deprecated. Please move to the pypdf library instead.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from dotenv import load_dotenv\n",
    "\n",
    "from evoagentx.agents.agent_manager import AgentManager\n",
    "from evoagentx.benchmark import HotPotQA\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.core.logging import logger\n",
    "from evoagentx.evaluators import Evaluator\n",
    "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
    "from evoagentx.optimizers import TextGradOptimizer\n",
    "from evoagentx.prompts import StringTemplate\n",
    "from evoagentx.workflow import SequentialWorkFlowGraph\n",
    "\n",
    "load_dotenv()\n",
    "\n",
    "class HotPotQASplits(HotPotQA):\n",
    "\n",
    "    def _load_data(self):\n",
    "        # load the original test data \n",
    "        super()._load_data()\n",
    "        # split the data into train, dev and test\n",
    "        import numpy as np \n",
    "        np.random.seed(42)\n",
    "        permutation = np.random.permutation(len(self._dev_data))\n",
    "        full_test_data = self._dev_data \n",
    "        # randomly select 10 samples for train, 40 for dev, and 100 for test\n",
    "        self._train_data = [full_test_data[idx] for idx in permutation[:10]]\n",
    "        self._dev_data = [full_test_data[idx] for idx in permutation[10:50]]\n",
    "        self._test_data = [full_test_data[idx] for idx in permutation[50:150]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4fc4aacd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-19 20:25:14.235\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.hotpotqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m51\u001b[0m - \u001b[1mloading HotPotQA data from /gpfs/radev/home/tl688/.evoagentx/data/hotpotqa/hotpot_train_v1.1.json ...\u001b[0m\n",
      "\u001b[32m2025-12-19 20:25:17.815\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.hotpotqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m51\u001b[0m - \u001b[1mloading HotPotQA data from /gpfs/radev/home/tl688/.evoagentx/data/hotpotqa/hotpot_dev_distractor_v1.json ...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "data = HotPotQASplits()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3ce3c923",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Gesellschaft mit beschränkter Haftung'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data._train_data[0]['answer']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "da31d473",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"VIVA Media AG changed it's name in 2004. What does their new acronym stand for?\""
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data._train_data[0]['question']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "508051c8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'_id': '5a7613c15542994ccc9186bf',\n",
       " 'answer': 'Gesellschaft mit beschränkter Haftung',\n",
       " 'question': \"VIVA Media AG changed it's name in 2004. What does their new acronym stand for?\",\n",
       " 'supporting_facts': [['VIVA Media', 0],\n",
       "  ['Gesellschaft mit beschränkter Haftung', 0]],\n",
       " 'context': [['Constantin Medien',\n",
       "   ['Constantin Medien AG (formerly EM.Entertainment and EM.TV & Merchandising AG, then EM.TV AG, and finally em.sport media ag) is a German media group, based in Ismaning near Munich, active in the area of sports, film and event marketing to medium-sized media companies.']],\n",
       "  ['VIVA Poland',\n",
       "   ['VIVA Polska (earlier \"VIVApolska!\")',\n",
       "    ' is a Polish 24h music and entertainment channel from Viacom International Media Networks Polska.',\n",
       "    ' The channel was officially launched on June 10, 2000 by the German VIVA Media AG.']],\n",
       "  ['Viva (UK and Ireland)',\n",
       "   ['Viva (stylised as VIVA) is a music television channel in the United Kingdom and Ireland, owned by VIVA Media and thereby Viacom International Media Networks Europe.',\n",
       "    ' The channel launched on 26 October 2009, replacing TMF.']],\n",
       "  ['Blic',\n",
       "   ['Blic (Cyrillic: Блиц, ] ) is a daily middle-market tabloid newspaper in Serbia.',\n",
       "    ' Founded in 1996, \"Blic\" is owned by Ringier Axel Springer Media AG, a joint venture between Ringier media corporation from Switzerland and Axel Springer AG from Germany.']],\n",
       "  ['Qontis',\n",
       "   ['Qontis is a Switzerland based online personal finance management (PFM) platform.',\n",
       "    ' The service is part of a commercial enterprise between the \"Neue Zürcher Zeitung\" media property and e-banking solutions provider Crealogix.',\n",
       "    ' The platform provides users with the ability to document and organize data from all instances of private income and expenditures.',\n",
       "    \" Qontis' CEO (chief executive officer) is Christian Bieri, who formerly served as the Austrian Country Manager and CEE for the Vienna branch of Avaloq Evolution AG.\",\n",
       "    \" The company's CMO (chief marketing officer) is Nils Reimelt, the former digital director at Ringier Axel Springer Media AG.\"]],\n",
       "  ['VIVA Media',\n",
       "   ['VIVA Media GmbH (until 2004 \"VIVA Media AG\") is a music television network originating from Germany.',\n",
       "    ' It was founded for broadcast of VIVA Germany as VIVA Media AG in 1993 and has been owned by their original concurrent Viacom, the parent company of MTV, since 2004.',\n",
       "    ' Viva channels exist in some European countries; the first spin-offs were launched in Poland and Switzerland in 2000.']],\n",
       "  ['ProSiebenSat.1 Media',\n",
       "   ['ProSiebenSat.1 Media SE (officially abbreviated as P7S1, formerly ProSiebenSat.1 Media AG) is a European mass media company, based in Germany.',\n",
       "    ' It operates free-to-air commercial TV channels, pay TV channels, radio stations and related print businesses.',\n",
       "    ' It was formed on October 2, 2000 by the merger of German TV broadcasters ProSieben Media AG (founded in 1989) and Sat.1 SatellitenFernsehen GmbH (founded in 1984 as PKS (Programmgesellschaft für Kabel- und Satellitenrundfunk)).',\n",
       "    ' The company is listed on the Frankfurt Stock Exchange and is a component of the DAX index.']],\n",
       "  ['Gesellschaft mit beschränkter Haftung',\n",
       "   ['A Gesellschaft mit beschränkter Haftung (] , abbreviated GmbH ] and also GesmbH in Austria) is a type of legal entity very common in Germany, Austria, Switzerland (where it is equivalent to a S.à r.l.) and Liechtenstein.',\n",
       "    ' In the United States, the equivalent type of entity is the limited liability company (LLC).',\n",
       "    ' The name of the GmbH form emphasizes the fact that the owners (\"Gesellschafter\", also known as members) of the entity are not personally liable for the company\\'s debts.',\n",
       "    ' \"GmbH\"s are considered legal persons under German and Austrian law.',\n",
       "    ' Other variations include mbH (used when the term \"Gesellschaft\" is part of the company name itself), and gGmbH (\"gemeinnützige\" GmbH) for non-profit companies.']],\n",
       "  ['Mix Megapol',\n",
       "   ['Mix Megapol is a private Swedish radio network controlled by ProSiebenSat.1 Media AG.',\n",
       "    ' It launched in 1993 under the name Skärgårdsradion (Archipelago Radio).',\n",
       "    ' Later that year the name was changed to Radio Megapol when the broadcasting permissions were auctioned out.',\n",
       "    ' In 1997 the word \"Mix\" was added and their slogan became \"The best mix of hits and oldies\".',\n",
       "    ' Mix Megapol is on air in 24 cities from Kiruna in the north to Malmö in the south.',\n",
       "    ' They have over two million listeners per week.',\n",
       "    ' Their target group is people aged between 25 and 45.']],\n",
       "  ['John M. Keller',\n",
       "   ['John M. Keller (born March 5, 1938) is an American educational psychologist.',\n",
       "    ' He is best known for his work on motivation in educational settings and in particular the ARCS model of instructional design.',\n",
       "    ' The four elements of the acronym stand for Attention, Relevance, Confidence and Satisfaction (ARCS).']]],\n",
       " 'type': 'bridge',\n",
       " 'level': 'hard'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data._train_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4580a4b1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "eacd5f0d",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'adata' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mNameError\u001b[39m                                 Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[16]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[43madata\u001b[49m \n",
      "\u001b[31mNameError\u001b[39m: name 'adata' is not defined"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "687d8d33",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "45e010eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"/gpfs/radev/project/zhao/hs2234/evox/QA/gene_perturb/qa/old_data/adamson_psa_single_qa.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "f06467c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"You are an expert in single-cell biology and functional genomics. In K562 cells, AARS is perturbed and the expression of ENSG00000092621 is measured. Does this perturbation cause a significant change in ENSG00000092621 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\""
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['question'].values[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "275fd8da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>You are an expert in single-cell biology and f...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>You are a domain expert in functional genomics...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>You are assisting with the interpretation of p...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>You are an expert in regulatory genomics. Cons...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>You are a specialist in gene perturbation expe...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2575</th>\n",
       "      <td>You are an expert in gene expression analysis....</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2576</th>\n",
       "      <td>You are a genomics expert evaluating perturbat...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2577</th>\n",
       "      <td>You are an expert analyst of perturbation data...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2578</th>\n",
       "      <td>You are a functional genomics specialist. In K...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2579</th>\n",
       "      <td>You are an expert in gene regulation studies. ...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2580 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               question answer\n",
       "0     You are an expert in single-cell biology and f...    Yes\n",
       "1     You are a domain expert in functional genomics...    Yes\n",
       "2     You are assisting with the interpretation of p...    Yes\n",
       "3     You are an expert in regulatory genomics. Cons...    Yes\n",
       "4     You are a specialist in gene perturbation expe...    Yes\n",
       "...                                                 ...    ...\n",
       "2575  You are an expert in gene expression analysis....    Yes\n",
       "2576  You are a genomics expert evaluating perturbat...    Yes\n",
       "2577  You are an expert analyst of perturbation data...    Yes\n",
       "2578  You are a functional genomics specialist. In K...    Yes\n",
       "2579  You are an expert in gene regulation studies. ...    Yes\n",
       "\n",
       "[2580 rows x 2 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "a6567e58",
   "metadata": {},
   "outputs": [],
   "source": [
    "map_dict = pd.read_pickle(\"./map_info_gene_adamson.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "c21167c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# more powerful model;\n",
    "# api/knowledge base query;\n",
    "# analyze system; "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "d93c7292",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ENSG00000228463': 'AP006222.2',\n",
       " 'ENSG00000272438': 'RP11-54O7.16',\n",
       " 'ENSG00000230699': 'RP11-54O7.1',\n",
       " 'ENSG00000223764': 'RP11-54O7.3',\n",
       " 'ENSG00000187634': 'SAMD11',\n",
       " 'ENSG00000188290': 'HES4',\n",
       " 'ENSG00000187608': 'ISG15',\n",
       " 'ENSG00000162571': 'TTLL10',\n",
       " 'ENSG00000186891': 'TNFRSF18',\n",
       " 'ENSG00000186827': 'TNFRSF4',\n",
       " 'ENSG00000078808': 'SDF4',\n",
       " 'ENSG00000221978': 'CCNL2',\n",
       " 'ENSG00000272455': 'RP4-758J18.13',\n",
       " 'ENSG00000215915': 'ATAD3C',\n",
       " 'ENSG00000272106': 'RP11-345P4.9',\n",
       " 'ENSG00000197530': 'MIB2',\n",
       " 'ENSG00000189409': 'MMP23B',\n",
       " 'ENSG00000169885': 'CALML6',\n",
       " 'ENSG00000271806': 'RP5-892K4.1',\n",
       " 'ENSG00000182873': 'RP11-181G12.2',\n",
       " 'ENSG00000269554': 'AL590822.2',\n",
       " 'ENSG00000272449': 'RP3-395M20.12',\n",
       " 'ENSG00000157873': 'TNFRSF14',\n",
       " 'ENSG00000238260': 'RP11-46F15.2',\n",
       " 'ENSG00000235169': 'SMIM1',\n",
       " 'ENSG00000236423': 'LINC01134',\n",
       " 'ENSG00000187017': 'ESPN',\n",
       " 'ENSG00000049249': 'TNFRSF9',\n",
       " 'ENSG00000116285': 'ERRFI1',\n",
       " 'ENSG00000142599': 'RERE',\n",
       " 'ENSG00000074800': 'ENO1',\n",
       " 'ENSG00000049239': 'H6PD',\n",
       " 'ENSG00000178585': 'CTNNBIP1',\n",
       " 'ENSG00000054523': 'KIF1B',\n",
       " 'ENSG00000271895': 'RP4-635E18.8',\n",
       " 'ENSG00000226849': 'RP4-635E18.7',\n",
       " 'ENSG00000132879': 'FBXO44',\n",
       " 'ENSG00000177000': 'MTHFR',\n",
       " 'ENSG00000171729': 'TMEM51',\n",
       " 'ENSG00000142634': 'EFHD2',\n",
       " 'ENSG00000142615': 'CELA2A',\n",
       " 'ENSG00000237301': 'RP4-680D5.2',\n",
       " 'ENSG00000162461': 'SLC25A34',\n",
       " 'ENSG00000162458': 'FBLIM1',\n",
       " 'ENSG00000179743': 'RP11-169K16.9',\n",
       " 'ENSG00000065526': 'SPEN',\n",
       " 'ENSG00000173641': 'HSPB7',\n",
       " 'ENSG00000142627': 'EPHA2',\n",
       " 'ENSG00000157191': 'NECAP2',\n",
       " 'ENSG00000261135': 'RP4-798A10.7',\n",
       " 'ENSG00000219481': 'NBPF1',\n",
       " 'ENSG00000227751': 'RP1-20B21.4',\n",
       " 'ENSG00000117148': 'ACTL8',\n",
       " 'ENSG00000230424': 'RP1-43E13.2',\n",
       " 'ENSG00000270136': 'MINOS1-NBL1',\n",
       " 'ENSG00000162542': 'TMCO4',\n",
       " 'ENSG00000187980': 'PLA2G2C',\n",
       " 'ENSG00000225986': 'UBXN10-AS1',\n",
       " 'ENSG00000117242': 'PINK1-AS',\n",
       " 'ENSG00000244038': 'DDOST',\n",
       " 'ENSG00000127483': 'HP1BP3',\n",
       " 'ENSG00000075151': 'EIF4G3',\n",
       " 'ENSG00000004487': 'KDM1A',\n",
       " 'ENSG00000125944': 'HNRNPR',\n",
       " 'ENSG00000249087': 'C1orf213',\n",
       " 'ENSG00000117318': 'ID3',\n",
       " 'ENSG00000179163': 'FUCA1',\n",
       " 'ENSG00000020633': 'RUNX3',\n",
       " 'ENSG00000117614': 'SYF2',\n",
       " 'ENSG00000117616': 'C1orf63',\n",
       " 'ENSG00000272432': 'RP3-465N24.6',\n",
       " 'ENSG00000188672': 'RHCE',\n",
       " 'ENSG00000233478': 'RP1-187B23.1',\n",
       " 'ENSG00000228172': 'RP1-317E23.3',\n",
       " 'ENSG00000223474': 'AL020996.1',\n",
       " 'ENSG00000272478': 'RP1-317E23.7',\n",
       " 'ENSG00000182749': 'PAQR7',\n",
       " 'ENSG00000117632': 'STMN1',\n",
       " 'ENSG00000158014': 'SLC30A2',\n",
       " 'ENSG00000188782': 'CATSPER4',\n",
       " 'ENSG00000142669': 'SH3BGRL3',\n",
       " 'ENSG00000169442': 'CD52',\n",
       " 'ENSG00000176092': 'AIM1L',\n",
       " 'ENSG00000131914': 'LIN28A',\n",
       " 'ENSG00000117682': 'DHDDS',\n",
       " 'ENSG00000198830': 'HMGN2',\n",
       " 'ENSG00000158246': 'FAM46B',\n",
       " 'ENSG00000142765': 'SYTL1',\n",
       " 'ENSG00000158195': 'WASF2',\n",
       " 'ENSG00000126709': 'IFI6',\n",
       " 'ENSG00000169403': 'PTAFR',\n",
       " 'ENSG00000270605': 'RP5-1092A3.4',\n",
       " 'ENSG00000130766': 'SESN2',\n",
       " 'ENSG00000197989': 'SNHG12',\n",
       " 'ENSG00000188060': 'RAB42',\n",
       " 'ENSG00000229388': 'RP11-442N24__B.1',\n",
       " 'ENSG00000162419': 'GMEB1',\n",
       " 'ENSG00000198492': 'YTHDF2',\n",
       " 'ENSG00000159023': 'EPB41',\n",
       " 'ENSG00000116350': 'SRSF4',\n",
       " 'ENSG00000227637': 'RP11-242O24.5',\n",
       " 'ENSG00000162511': 'LAPTM5',\n",
       " 'ENSG00000162512': 'SDC3',\n",
       " 'ENSG00000134644': 'PUM1',\n",
       " 'ENSG00000121769': 'FABP3',\n",
       " 'ENSG00000142910': 'TINAGL1',\n",
       " 'ENSG00000134668': 'SPOCD1',\n",
       " 'ENSG00000121774': 'KHDRBS1',\n",
       " 'ENSG00000084652': 'TXLNA',\n",
       " 'ENSG00000160050': 'CCDC28B',\n",
       " 'ENSG00000182866': 'LCK',\n",
       " 'ENSG00000175130': 'MARCKSL1',\n",
       " 'ENSG00000162526': 'TSSK3',\n",
       " 'ENSG00000225828': 'FAM229A',\n",
       " 'ENSG00000160058': 'BSDC1',\n",
       " 'ENSG00000162520': 'SYNC',\n",
       " 'ENSG00000224409': 'RP11-114B7.6',\n",
       " 'ENSG00000162522': 'KIAA1522',\n",
       " 'ENSG00000121900': 'TMEM54',\n",
       " 'ENSG00000197056': 'ZMYM1',\n",
       " 'ENSG00000116560': 'SFPQ',\n",
       " 'ENSG00000146463': 'ZMYM4',\n",
       " 'ENSG00000092853': 'CLSPN',\n",
       " 'ENSG00000134698': 'AGO4',\n",
       " 'ENSG00000092847': 'AGO1',\n",
       " 'ENSG00000054118': 'THRAP3',\n",
       " 'ENSG00000142694': 'EVA1B',\n",
       " 'ENSG00000116885': 'OSCP1',\n",
       " 'ENSG00000119535': 'CSF3R',\n",
       " 'ENSG00000233621': 'LINC01137',\n",
       " 'ENSG00000163874': 'ZC3H12A',\n",
       " 'ENSG00000163879': 'DNALI1',\n",
       " 'ENSG00000134690': 'CDCA8',\n",
       " 'ENSG00000183386': 'FHL3',\n",
       " 'ENSG00000185668': 'POU3F1',\n",
       " 'ENSG00000224592': 'RP5-884C9.2',\n",
       " 'ENSG00000116954': 'RRAGC',\n",
       " 'ENSG00000158315': 'RHBDL2',\n",
       " 'ENSG00000127603': 'MACF1',\n",
       " 'ENSG00000255103': 'KIAA0754',\n",
       " 'ENSG00000261798': 'RP1-118J21.25',\n",
       " 'ENSG00000131236': 'CAP1',\n",
       " 'ENSG00000117000': 'RLF',\n",
       " 'ENSG00000188800': 'TMCO2',\n",
       " 'ENSG00000259943': 'RP1-39G22.7',\n",
       " 'ENSG00000049089': 'COL9A2',\n",
       " 'ENSG00000084070': 'SMAP2',\n",
       " 'ENSG00000117016': 'RIMS3',\n",
       " 'ENSG00000272145': 'NFYC-AS1',\n",
       " 'ENSG00000171790': 'SLFNL1',\n",
       " 'ENSG00000204060': 'FOXO6',\n",
       " 'ENSG00000127124': 'HIVEP3',\n",
       " 'ENSG00000127125': 'PPCS',\n",
       " 'ENSG00000186409': 'CCDC30',\n",
       " 'ENSG00000117385': 'LEPRE1',\n",
       " 'ENSG00000227533': 'SLC2A1-AS1',\n",
       " 'ENSG00000117399': 'CDC20',\n",
       " 'ENSG00000237950': 'RP11-7O11.3',\n",
       " 'ENSG00000117410': 'ATP6V0B',\n",
       " 'ENSG00000198520': 'C1orf228',\n",
       " 'ENSG00000142945': 'KIF2C',\n",
       " 'ENSG00000225721': 'RP11-269F19.2',\n",
       " 'ENSG00000142937': 'RPS8',\n",
       " 'ENSG00000188396': 'TCTEX1D4',\n",
       " 'ENSG00000222009': 'BTBD19',\n",
       " 'ENSG00000117425': 'PTCH2',\n",
       " 'ENSG00000070785': 'EIF2B3',\n",
       " 'ENSG00000162415': 'ZSWIM5',\n",
       " 'ENSG00000070759': 'TESK2',\n",
       " 'ENSG00000117450': 'PRDX1',\n",
       " 'ENSG00000132780': 'NASP',\n",
       " 'ENSG00000159588': 'CCDC17',\n",
       " 'ENSG00000117461': 'PIK3R3',\n",
       " 'ENSG00000225667': 'LINC00505',\n",
       " 'ENSG00000159658': 'EFCAB14',\n",
       " 'ENSG00000225506': 'CYP4A22-AS1',\n",
       " 'ENSG00000224805': 'LINC00853',\n",
       " 'ENSG00000162366': 'PDZK1IP1',\n",
       " 'ENSG00000186790': 'FOXE3',\n",
       " 'ENSG00000237424': 'FOXD2-AS1',\n",
       " 'ENSG00000271782': 'RP5-850O15.4',\n",
       " 'ENSG00000123080': 'CDKN2C',\n",
       " 'ENSG00000203356': 'RP11-296A18.3',\n",
       " 'ENSG00000123091': 'RNF11',\n",
       " 'ENSG00000261664': 'RP11-275F13.1',\n",
       " 'ENSG00000085832': 'EPS15',\n",
       " 'ENSG00000227070': 'RP11-191G24.1',\n",
       " 'ENSG00000078618': 'NRD1',\n",
       " 'ENSG00000266993': 'RP4-657D16.3',\n",
       " 'ENSG00000117862': 'TXNDC12',\n",
       " 'ENSG00000223390': 'RP11-91A18.4',\n",
       " 'ENSG00000157077': 'ZFYVE9',\n",
       " 'ENSG00000134744': 'ZCCHC11',\n",
       " 'ENSG00000121310': 'ECHDC2',\n",
       " 'ENSG00000230138': 'RP11-117D22.2',\n",
       " 'ENSG00000116209': 'TMEM59',\n",
       " 'ENSG00000256407': 'RP11-446E24.4',\n",
       " 'ENSG00000162390': 'ACOT11',\n",
       " 'ENSG00000143001': 'TMEM61',\n",
       " 'ENSG00000162601': 'MYSM1',\n",
       " 'ENSG00000177606': 'JUN',\n",
       " 'ENSG00000226476': 'RP11-776H12.1',\n",
       " 'ENSG00000237853': 'RP5-833A20.1',\n",
       " 'ENSG00000162607': 'USP1',\n",
       " 'ENSG00000230798': 'RP4-792G4.2',\n",
       " 'ENSG00000162434': 'JAK1',\n",
       " 'ENSG00000231485': 'RP4-535B20.1',\n",
       " 'ENSG00000213625': 'LEPROT',\n",
       " 'ENSG00000152763': 'WDR78',\n",
       " 'ENSG00000198160': 'MIER1',\n",
       " 'ENSG00000116717': 'GADD45A',\n",
       " 'ENSG00000232284': 'GNG12-AS1',\n",
       " 'ENSG00000024526': 'DEPDC1',\n",
       " 'ENSG00000116761': 'CTH',\n",
       " 'ENSG00000050628': 'PTGER3',\n",
       " 'ENSG00000235782': 'RP3-333A15.1',\n",
       " 'ENSG00000272864': 'RP11-17E13.2',\n",
       " 'ENSG00000137968': 'SLC44A5',\n",
       " 'ENSG00000162613': 'FUBP1',\n",
       " 'ENSG00000162616': 'DNAJB4',\n",
       " 'ENSG00000137941': 'TTLL7',\n",
       " 'ENSG00000271576': 'RP11-486G15.2',\n",
       " 'ENSG00000142875': 'PRKACB',\n",
       " 'ENSG00000117151': 'CTBS',\n",
       " 'ENSG00000117155': 'SSX2IP',\n",
       " 'ENSG00000142867': 'BCL10',\n",
       " 'ENSG00000117174': 'ZNHIT6',\n",
       " 'ENSG00000122417': 'ODF2L',\n",
       " 'ENSG00000016490': 'CLCA1',\n",
       " 'ENSG00000097033': 'SH3GLB1',\n",
       " 'ENSG00000267734': 'RP4-604K5.3',\n",
       " 'ENSG00000224198': 'RP4-604K5.2',\n",
       " 'ENSG00000143013': 'LMO4',\n",
       " 'ENSG00000065243': 'PKN2',\n",
       " 'ENSG00000137947': 'GTF2B',\n",
       " 'ENSG00000117228': 'GBP1',\n",
       " 'ENSG00000162645': 'GBP2',\n",
       " 'ENSG00000197147': 'LRRC8B',\n",
       " 'ENSG00000162664': 'ZNF326',\n",
       " 'ENSG00000069712': 'KIAA1107',\n",
       " 'ENSG00000122484': 'RPAP2',\n",
       " 'ENSG00000067208': 'EVI5',\n",
       " 'ENSG00000137942': 'FNBP1L',\n",
       " 'ENSG00000137936': 'BCAR3',\n",
       " 'ENSG00000067334': 'DNTTIP2',\n",
       " 'ENSG00000122481': 'RWDD3',\n",
       " 'ENSG00000156869': 'FRRS1',\n",
       " 'ENSG00000122477': 'LRRC39',\n",
       " 'ENSG00000137992': 'DBT',\n",
       " 'ENSG00000170989': 'S1PR1',\n",
       " 'ENSG00000273382': 'RP5-1065J22.8',\n",
       " 'ENSG00000270066': 'SCARNA2',\n",
       " 'ENSG00000031698': 'SARS',\n",
       " 'ENSG00000134222': 'PSRC1',\n",
       " 'ENSG00000221986': 'MYBPHL',\n",
       " 'ENSG00000134243': 'SORT1',\n",
       " 'ENSG00000134201': 'GSTM5',\n",
       " 'ENSG00000134202': 'GSTM3',\n",
       " 'ENSG00000261055': 'RP11-195M16.3',\n",
       " 'ENSG00000184371': 'CSF1',\n",
       " 'ENSG00000273373': 'RP5-1074L1.4',\n",
       " 'ENSG00000143119': 'CD53',\n",
       " 'ENSG00000232811': 'RP11-96K19.2',\n",
       " 'ENSG00000121931': 'LRIF1',\n",
       " 'ENSG00000162777': 'DENND2D',\n",
       " 'ENSG00000064886': 'CHI3L2',\n",
       " 'ENSG00000085465': 'OVGP1',\n",
       " 'ENSG00000143110': 'C1orf162',\n",
       " 'ENSG00000197852': 'FAM212B',\n",
       " 'ENSG00000143079': 'CTTNBP2NL',\n",
       " 'ENSG00000007341': 'ST7L',\n",
       " 'ENSG00000225075': 'RP11-426L16.3',\n",
       " 'ENSG00000155366': 'RHOC',\n",
       " 'ENSG00000215866': 'RP11-426L16.8',\n",
       " 'ENSG00000238198': 'RP11-31F15.2',\n",
       " 'ENSG00000116793': 'PHTF1',\n",
       " 'ENSG00000226167': 'AP4B1-AS1',\n",
       " 'ENSG00000235527': 'RP5-1073O3.7',\n",
       " 'ENSG00000134207': 'SYT6',\n",
       " 'ENSG00000197323': 'TRIM33',\n",
       " 'ENSG00000163393': 'SLC22A15',\n",
       " 'ENSG00000269279': 'AL136376.1',\n",
       " 'ENSG00000224950': 'RP5-1086K13.1',\n",
       " 'ENSG00000116815': 'CD58',\n",
       " 'ENSG00000198162': 'MAN1A2',\n",
       " 'ENSG00000183508': 'FAM46C',\n",
       " 'ENSG00000065183': 'WDR3',\n",
       " 'ENSG00000231365': 'RP11-418J17.1',\n",
       " 'ENSG00000092621': 'PHGDH',\n",
       " 'ENSG00000198019': 'FCGR1B',\n",
       " 'ENSG00000230880': 'RP11-417J8.3',\n",
       " 'ENSG00000234978': 'RP11-423O2.5',\n",
       " 'ENSG00000242569': 'RP11-435B5.3',\n",
       " 'ENSG00000236943': 'RP11-640M9.1',\n",
       " 'ENSG00000178104': 'PDE4DIP',\n",
       " 'ENSG00000268172': 'AL590452.1',\n",
       " 'ENSG00000163386': 'NBPF10',\n",
       " 'ENSG00000168509': 'HFE2',\n",
       " 'ENSG00000117289': 'TXNIP',\n",
       " 'ENSG00000121851': 'POLR3GL',\n",
       " 'ENSG00000117281': 'CD160',\n",
       " 'ENSG00000174827': 'PDZK1',\n",
       " 'ENSG00000237188': 'RP11-337C18.8',\n",
       " 'ENSG00000271721': 'RP11-337C18.9',\n",
       " 'ENSG00000231196': 'RP11-495P10.8',\n",
       " 'ENSG00000272824': 'RP6-74O6.6',\n",
       " 'ENSG00000243452': 'NBPF15',\n",
       " 'ENSG00000203827': 'NBPF16',\n",
       " 'ENSG00000232527': 'RP11-14N7.2',\n",
       " 'ENSG00000229801': 'RP11-353N4.1',\n",
       " 'ENSG00000150337': 'FCGR1A',\n",
       " 'ENSG00000272993': 'RP11-196G18.24',\n",
       " 'ENSG00000184678': 'HIST2H2BE',\n",
       " 'ENSG00000184260': 'HIST2H2AC',\n",
       " 'ENSG00000159164': 'SV2A',\n",
       " 'ENSG00000014914': 'MTMR11',\n",
       " 'ENSG00000023902': 'PLEKHO1',\n",
       " 'ENSG00000143401': 'ANP32E',\n",
       " 'ENSG00000271845': 'RP4-790G17.7',\n",
       " 'ENSG00000117360': 'PRPF3',\n",
       " 'ENSG00000163125': 'RPRD2',\n",
       " 'ENSG00000143369': 'ECM1',\n",
       " 'ENSG00000228126': 'LINC00568',\n",
       " 'ENSG00000237781': 'RP11-54A4.2',\n",
       " 'ENSG00000143384': 'MCL1',\n",
       " 'ENSG00000163131': 'CTSS',\n",
       " 'ENSG00000143387': 'CTSK',\n",
       " 'ENSG00000143409': 'FAM63A',\n",
       " 'ENSG00000143443': 'C1orf56',\n",
       " 'ENSG00000197622': 'CDC42SE1',\n",
       " 'ENSG00000213190': 'MLLT11',\n",
       " 'ENSG00000163154': 'TNFAIP8L2',\n",
       " 'ENSG00000163155': 'LYSMD1',\n",
       " 'ENSG00000159352': 'PSMD4',\n",
       " 'ENSG00000143416': 'SELENBP1',\n",
       " 'ENSG00000143367': 'TUFT1',\n",
       " 'ENSG00000227045': 'RP11-98D18.1',\n",
       " 'ENSG00000269489': 'RP11-98D18.17',\n",
       " 'ENSG00000197747': 'S100A10',\n",
       " 'ENSG00000163191': 'S100A11',\n",
       " 'ENSG00000143520': 'FLG2',\n",
       " 'ENSG00000203783': 'PRR9',\n",
       " 'ENSG00000163218': 'PGLYRP4',\n",
       " 'ENSG00000163220': 'S100A9',\n",
       " 'ENSG00000196420': 'S100A5',\n",
       " 'ENSG00000196154': 'S100A4',\n",
       " 'ENSG00000188015': 'S100A3',\n",
       " 'ENSG00000196754': 'S100A2',\n",
       " 'ENSG00000188643': 'S100A16',\n",
       " 'ENSG00000189334': 'S100A14',\n",
       " 'ENSG00000189171': 'S100A13',\n",
       " 'ENSG00000160678': 'S100A1',\n",
       " 'ENSG00000143614': 'GATAD2B',\n",
       " 'ENSG00000273026': 'RP11-422P24.10',\n",
       " 'ENSG00000143545': 'RAB13',\n",
       " 'ENSG00000177954': 'RPS27',\n",
       " 'ENSG00000143549': 'TPM3',\n",
       " 'ENSG00000143575': 'HAX1',\n",
       " 'ENSG00000143595': 'AQP10',\n",
       " 'ENSG00000160712': 'IL6R',\n",
       " 'ENSG00000160710': 'ADAR',\n",
       " 'ENSG00000270361': 'RP11-307C12.13',\n",
       " 'ENSG00000163346': 'PBXIP1',\n",
       " 'ENSG00000173207': 'CKS1B',\n",
       " 'ENSG00000169242': 'EFNA1',\n",
       " 'ENSG00000116539': 'ASH1L',\n",
       " 'ENSG00000132718': 'SYT11',\n",
       " 'ENSG00000143622': 'RIT1',\n",
       " 'ENSG00000132680': 'KIAA0907',\n",
       " 'ENSG00000116584': 'ARHGEF2',\n",
       " 'ENSG00000273002': 'RP11-336K24.12',\n",
       " 'ENSG00000163479': 'SSR2',\n",
       " 'ENSG00000132698': 'RAB25',\n",
       " 'ENSG00000160789': 'LMNA',\n",
       " 'ENSG00000160781': 'PAQR6',\n",
       " 'ENSG00000125462': 'C1orf61',\n",
       " 'ENSG00000236947': 'RP11-98G7.1',\n",
       " 'ENSG00000116604': 'MEF2D',\n",
       " 'ENSG00000160818': 'GPATCH4',\n",
       " 'ENSG00000272068': 'RP11-284F21.9',\n",
       " 'ENSG00000132692': 'BCAN',\n",
       " 'ENSG00000272405': 'RP11-284F21.10',\n",
       " 'ENSG00000229953': 'RP11-284F21.7',\n",
       " 'ENSG00000132688': 'NES',\n",
       " 'ENSG00000143320': 'CRABP2',\n",
       " 'ENSG00000027869': 'SH2D2A',\n",
       " 'ENSG00000198400': 'NTRK1',\n",
       " 'ENSG00000132694': 'ARHGEF11',\n",
       " 'ENSG00000117036': 'ETV3',\n",
       " 'ENSG00000271736': 'RP11-85G21.3',\n",
       " 'ENSG00000163565': 'IFI16',\n",
       " 'ENSG00000158716': 'DUSP23',\n",
       " 'ENSG00000213085': 'CCDC19',\n",
       " 'ENSG00000018625': 'ATP1A2',\n",
       " 'ENSG00000132681': 'ATP1A4',\n",
       " 'ENSG00000162734': 'PEA15',\n",
       " 'ENSG00000228606': 'RP11-574F21.2',\n",
       " 'ENSG00000122218': 'COPA',\n",
       " 'ENSG00000171786': 'NHLH1',\n",
       " 'ENSG00000162739': 'SLAMF6',\n",
       " 'ENSG00000066294': 'CD84',\n",
       " 'ENSG00000117090': 'SLAMF1',\n",
       " 'ENSG00000122223': 'CD244',\n",
       " 'ENSG00000179914': 'ITLN1',\n",
       " 'ENSG00000158764': 'ITLN2',\n",
       " 'ENSG00000215845': 'TSTD1',\n",
       " 'ENSG00000186517': 'ARHGAP30',\n",
       " 'ENSG00000143217': 'PVRL4',\n",
       " 'ENSG00000158859': 'ADAMTS4',\n",
       " 'ENSG00000158869': 'FCER1G',\n",
       " 'ENSG00000158887': 'MPZ',\n",
       " 'ENSG00000235477': 'RP11-122G18.5',\n",
       " 'ENSG00000273075': 'RP11-122G18.8',\n",
       " 'ENSG00000143226': 'FCGR2A',\n",
       " 'ENSG00000173110': 'HSPA6',\n",
       " 'ENSG00000118217': 'ATF6',\n",
       " 'ENSG00000198929': 'NOS1AP',\n",
       " 'ENSG00000152332': 'UHMK1',\n",
       " 'ENSG00000143228': 'NUF2',\n",
       " 'ENSG00000185630': 'PBX1',\n",
       " 'ENSG00000143183': 'TMCO1',\n",
       " 'ENSG00000234142': 'RP11-276E17.2',\n",
       " 'ENSG00000143190': 'POU2F1',\n",
       " 'ENSG00000143162': 'CREG1',\n",
       " 'ENSG00000198771': 'RCSD1',\n",
       " 'ENSG00000232194': 'RP1-313L4.3',\n",
       " 'ENSG00000143158': 'MPC2',\n",
       " 'ENSG00000228697': 'RP5-968D22.1',\n",
       " 'ENSG00000143153': 'ATP1B1',\n",
       " 'ENSG00000075945': 'KIFAP3',\n",
       " 'ENSG00000076258': 'FMO4',\n",
       " 'ENSG00000117523': 'PRRC2C',\n",
       " 'ENSG00000094975': 'SUCO',\n",
       " 'ENSG00000270084': 'GAS5-AS1',\n",
       " 'ENSG00000152061': 'RABGAP1L',\n",
       " 'ENSG00000203737': 'GPR52',\n",
       " 'ENSG00000116161': 'CACYBP',\n",
       " 'ENSG00000236021': 'RP11-195C7.1',\n",
       " 'ENSG00000213057': 'C1orf220',\n",
       " 'ENSG00000273384': 'RP5-1098D14.1',\n",
       " 'ENSG00000273062': 'RP11-428K3.1',\n",
       " 'ENSG00000116191': 'RALGPS2',\n",
       " 'ENSG00000143322': 'ABL2',\n",
       " 'ENSG00000162779': 'AXDND1',\n",
       " 'ENSG00000169905': 'TOR1AIP2',\n",
       " 'ENSG00000143337': 'TOR1AIP1',\n",
       " 'ENSG00000135837': 'CEP350',\n",
       " 'ENSG00000116260': 'QSOX1',\n",
       " 'ENSG00000135835': 'KIAA1614',\n",
       " 'ENSG00000135823': 'STX6',\n",
       " 'ENSG00000153029': 'MR1',\n",
       " 'ENSG00000162783': 'IER5',\n",
       " 'ENSG00000272198': 'RP11-309G3.3',\n",
       " 'ENSG00000135828': 'RNASEL',\n",
       " 'ENSG00000143333': 'RGS16',\n",
       " 'ENSG00000261504': 'RP11-317P15.4',\n",
       " 'ENSG00000135838': 'NPL',\n",
       " 'ENSG00000135829': 'DHX9',\n",
       " 'ENSG00000116698': 'SMG7',\n",
       " 'ENSG00000116701': 'NCF2',\n",
       " 'ENSG00000162704': 'ARPC5',\n",
       " 'ENSG00000198756': 'COLGALT2',\n",
       " 'ENSG00000198860': 'TSEN15',\n",
       " 'ENSG00000135842': 'FAM129A',\n",
       " 'ENSG00000047410': 'TPR',\n",
       " 'ENSG00000116711': 'PLA2G4A',\n",
       " 'ENSG00000134371': 'CDC73',\n",
       " 'ENSG00000162630': 'B3GALT2',\n",
       " 'ENSG00000232077': 'LINC01031',\n",
       " 'ENSG00000231714': 'RP11-476H20.1',\n",
       " 'ENSG00000000971': 'CFH',\n",
       " 'ENSG00000116785': 'CFHR3',\n",
       " 'ENSG00000066279': 'ASPM',\n",
       " 'ENSG00000177888': 'ZBTB41',\n",
       " 'ENSG00000081237': 'PTPRC',\n",
       " 'ENSG00000203721': 'LINC00862',\n",
       " 'ENSG00000162702': 'ZNF281',\n",
       " 'ENSG00000118193': 'KIF14',\n",
       " 'ENSG00000260088': 'RP11-92G12.3',\n",
       " 'ENSG00000118200': 'CAMSAP2',\n",
       " 'ENSG00000223396': 'RPS10P7',\n",
       " 'ENSG00000134369': 'NAV1',\n",
       " 'ENSG00000231871': 'IPO9-AS1',\n",
       " 'ENSG00000198892': 'SHISA4',\n",
       " 'ENSG00000223774': 'RP11-307B6.3',\n",
       " 'ENSG00000163435': 'ELF3',\n",
       " 'ENSG00000143862': 'ARL8A',\n",
       " 'ENSG00000143851': 'PTPN7',\n",
       " 'ENSG00000077152': 'UBE2T',\n",
       " 'ENSG00000077157': 'PPP1R12B',\n",
       " 'ENSG00000117139': 'KDM5B',\n",
       " 'ENSG00000143847': 'PPFIA4',\n",
       " 'ENSG00000133063': 'CHIT1',\n",
       " 'ENSG00000231507': 'RP11-134P9.3',\n",
       " 'ENSG00000159388': 'BTG2',\n",
       " 'ENSG00000188783': 'PRELP',\n",
       " 'ENSG00000058668': 'ATP2B4',\n",
       " 'ENSG00000058673': 'ZC3H11A',\n",
       " 'ENSG00000257315': 'ZBED6',\n",
       " 'ENSG00000143842': 'SOX13',\n",
       " 'ENSG00000143850': 'PLEKHA6',\n",
       " 'ENSG00000198625': 'MDM4',\n",
       " 'ENSG00000240710': 'RP11-430C7.4',\n",
       " 'ENSG00000163531': 'NFASC',\n",
       " 'ENSG00000133069': 'TMCC2',\n",
       " 'ENSG00000163545': 'NUAK2',\n",
       " 'ENSG00000162873': 'KLHDC8A',\n",
       " 'ENSG00000117266': 'CDK18',\n",
       " 'ENSG00000069275': 'NUCKS1',\n",
       " 'ENSG00000268313': 'AC119673.1',\n",
       " 'ENSG00000229874': 'RP11-312O7.2',\n",
       " 'ENSG00000226780': 'RP11-38J22.3',\n",
       " 'ENSG00000196533': 'C1orf186',\n",
       " 'ENSG00000196188': 'CTSE',\n",
       " 'ENSG00000143466': 'IKBKE',\n",
       " 'ENSG00000136653': 'RASSF5',\n",
       " 'ENSG00000162889': 'MAPKAPK2',\n",
       " 'ENSG00000162892': 'IL24',\n",
       " 'ENSG00000162896': 'PIGR',\n",
       " 'ENSG00000180667': 'YOD1',\n",
       " 'ENSG00000196352': 'CD55',\n",
       " 'ENSG00000117322': 'CR2',\n",
       " 'ENSG00000197721': 'CR1L',\n",
       " 'ENSG00000117335': 'CD46',\n",
       " 'ENSG00000117650': 'NEK2',\n",
       " 'ENSG00000123684': 'LPGAT1',\n",
       " 'ENSG00000143476': 'DTL',\n",
       " 'ENSG00000226251': 'RP11-15I11.3',\n",
       " 'ENSG00000066027': 'PPP2R5A',\n",
       " 'ENSG00000234915': 'RP11-384C4.7',\n",
       " 'ENSG00000117691': 'NENF',\n",
       " 'ENSG00000162772': 'ATF3',\n",
       " 'ENSG00000203705': 'TATDN3',\n",
       " 'ENSG00000136643': 'RPS6KC1',\n",
       " 'ENSG00000117707': 'PROX1',\n",
       " 'ENSG00000152104': 'PTPN14',\n",
       " 'ENSG00000117724': 'CENPF',\n",
       " 'ENSG00000232480': 'RP11-224O19.2',\n",
       " 'ENSG00000136628': 'EPRS',\n",
       " 'ENSG00000067704': 'IARS2',\n",
       " 'ENSG00000117791': 'MARC2',\n",
       " 'ENSG00000257551': 'HLA-AS1',\n",
       " 'ENSG00000143507': 'DUSP10',\n",
       " 'ENSG00000154305': 'MIA3',\n",
       " 'ENSG00000272750': 'RP11-378J18.8',\n",
       " 'ENSG00000162819': 'BROX',\n",
       " 'ENSG00000162909': 'CAPN2',\n",
       " 'ENSG00000143514': 'TP53BP2',\n",
       " 'ENSG00000162923': 'WDR26',\n",
       " 'ENSG00000143786': 'CNIH3',\n",
       " 'ENSG00000143815': 'LBR',\n",
       " 'ENSG00000242861': 'RP11-285F7.2',\n",
       " 'ENSG00000196187': 'TMEM63A',\n",
       " 'ENSG00000243709': 'LEFTY1',\n",
       " 'ENSG00000143768': 'LEFTY2',\n",
       " 'ENSG00000143751': 'SDE2',\n",
       " 'ENSG00000182827': 'ACBD3',\n",
       " 'ENSG00000143799': 'PARP1',\n",
       " 'ENSG00000143776': 'CDC42BPA',\n",
       " 'ENSG00000181450': 'ZNF678',\n",
       " 'ENSG00000143816': 'WNT9A',\n",
       " 'ENSG00000162913': 'C1orf145',\n",
       " 'ENSG00000154358': 'OBSCN',\n",
       " 'ENSG00000270104': 'RP11-245P10.6',\n",
       " 'ENSG00000116574': 'RHOU',\n",
       " 'ENSG00000177788': 'RP5-1061H20.4',\n",
       " 'ENSG00000237481': 'RP4-803J11.2',\n",
       " 'ENSG00000143641': 'GALNT2',\n",
       " 'ENSG00000224407': 'RP5-956O18.3',\n",
       " 'ENSG00000182118': 'FAM89A',\n",
       " 'ENSG00000119283': 'TRIM67',\n",
       " 'ENSG00000212916': 'MAP10',\n",
       " 'ENSG00000183780': 'SLC35F3',\n",
       " 'ENSG00000168264': 'IRF2BP2',\n",
       " 'ENSG00000227630': 'LINC01132',\n",
       " 'ENSG00000054267': 'ARID4B',\n",
       " 'ENSG00000168243': 'GNG4',\n",
       " 'ENSG00000086619': 'ERO1LB',\n",
       " 'ENSG00000116977': 'LGALS8',\n",
       " 'ENSG00000119285': 'HEATR1',\n",
       " 'ENSG00000077522': 'ACTN2',\n",
       " 'ENSG00000198626': 'RYR2',\n",
       " 'ENSG00000133019': 'CHRM3',\n",
       " 'ENSG00000117009': 'KMO',\n",
       " 'ENSG00000054277': 'OPN3',\n",
       " 'ENSG00000203668': 'CHML',\n",
       " 'ENSG00000174371': 'EXO1',\n",
       " 'ENSG00000143702': 'CEP170',\n",
       " 'ENSG00000054282': 'SDCCAG8',\n",
       " 'ENSG00000117020': 'AKT3',\n",
       " 'ENSG00000179456': 'ZBTB18',\n",
       " 'ENSG00000173728': 'C1orf100',\n",
       " 'ENSG00000188206': 'HNRNPU-AS1',\n",
       " 'ENSG00000153187': 'HNRNPU',\n",
       " 'ENSG00000185420': 'SMYD3',\n",
       " 'ENSG00000242042': 'RP11-690C23.2',\n",
       " 'ENSG00000162852': 'CNST',\n",
       " 'ENSG00000153207': 'AHCTF1',\n",
       " 'ENSG00000135747': 'ZNF670',\n",
       " 'ENSG00000196418': 'ZNF124',\n",
       " 'ENSG00000232347': 'RP11-488L18.8',\n",
       " 'ENSG00000177212': 'OR2T33',\n",
       " 'ENSG00000198104': 'OR2T6',\n",
       " 'ENSG00000171161': 'ZNF672',\n",
       " 'ENSG00000225234': 'TRAPPC12-AS1',\n",
       " 'ENSG00000225964': 'AC017076.5',\n",
       " 'ENSG00000134321': 'RSAD2',\n",
       " 'ENSG00000271947': 'RP11-439M11.1',\n",
       " 'ENSG00000236008': 'AC011747.4',\n",
       " 'ENSG00000115738': 'ID2',\n",
       " 'ENSG00000134313': 'KIDINS220',\n",
       " 'ENSG00000151693': 'ASAP2',\n",
       " 'ENSG00000134330': 'IAH1',\n",
       " 'ENSG00000151694': 'ADAM17',\n",
       " 'ENSG00000271855': 'RP11-214N9.1',\n",
       " 'ENSG00000272524': 'RP11-254F7.4',\n",
       " 'ENSG00000171848': 'RRM2',\n",
       " 'ENSG00000115756': 'HPCAL1',\n",
       " 'ENSG00000115758': 'ODC1',\n",
       " 'ENSG00000243819': 'RN7SL832P',\n",
       " 'ENSG00000143882': 'ATP6V1C2',\n",
       " 'ENSG00000143870': 'PDIA6',\n",
       " 'ENSG00000134318': 'ROCK2',\n",
       " 'ENSG00000224177': 'LINC00570',\n",
       " 'ENSG00000134324': 'LPIN1',\n",
       " 'ENSG00000224184': 'AC096559.1',\n",
       " 'ENSG00000071575': 'TRIB2',\n",
       " 'ENSG00000151779': 'NBAS',\n",
       " 'ENSG00000233718': 'MYCNOS',\n",
       " 'ENSG00000134323': 'MYCN',\n",
       " 'ENSG00000197872': 'FAM49A',\n",
       " 'ENSG00000163029': 'SMC6',\n",
       " 'ENSG00000236204': 'AC092594.1',\n",
       " 'ENSG00000183891': 'TTC32',\n",
       " 'ENSG00000271991': 'RP11-79O8.1',\n",
       " 'ENSG00000115884': 'SDC1',\n",
       " 'ENSG00000143878': 'RHOB',\n",
       " 'ENSG00000260396': 'AC012065.7',\n",
       " 'ENSG00000119771': 'KLHL29',\n",
       " 'ENSG00000119778': 'ATAD2B',\n",
       " 'ENSG00000163026': 'C2orf44',\n",
       " 'ENSG00000219626': 'FAM228B',\n",
       " 'ENSG00000186453': 'FAM228A',\n",
       " 'ENSG00000198399': 'ITSN2',\n",
       " 'ENSG00000138031': 'ADCY3',\n",
       " 'ENSG00000272048': 'RP11-458N5.1',\n",
       " 'ENSG00000143970': 'ASXL2',\n",
       " 'ENSG00000115163': 'CENPA',\n",
       " 'ENSG00000231636': 'AGBL5-AS1',\n",
       " 'ENSG00000272148': 'RP11-195B17.1',\n",
       " 'ENSG00000228474': 'OST4',\n",
       " 'ENSG00000138028': 'CGREF1',\n",
       " 'ENSG00000084774': 'CAD',\n",
       " 'ENSG00000163794': 'UCN',\n",
       " 'ENSG00000115211': 'EIF2B4',\n",
       " 'ENSG00000115216': 'NRBP1',\n",
       " 'ENSG00000157992': 'KRTCAP3',\n",
       " 'ENSG00000115226': 'FNDC4',\n",
       " 'ENSG00000084734': 'GCKR',\n",
       " 'ENSG00000221843': 'C2orf16',\n",
       " 'ENSG00000075426': 'FOSL2',\n",
       " 'ENSG00000270210': 'RP11-373D23.3',\n",
       " 'ENSG00000163803': 'PLB1',\n",
       " 'ENSG00000213639': 'PPP1CB',\n",
       " 'ENSG00000163806': 'SPDYA',\n",
       " 'ENSG00000163811': 'WDR43',\n",
       " 'ENSG00000189350': 'FAM179A',\n",
       " 'ENSG00000119801': 'YPEL5',\n",
       " 'ENSG00000213626': 'LBH',\n",
       " 'ENSG00000271228': 'CTC-336P14.1',\n",
       " 'ENSG00000021574': 'SPAST',\n",
       " 'ENSG00000272716': 'RP11-563N4.1',\n",
       " 'ENSG00000272754': 'AL133245.2',\n",
       " 'ENSG00000115760': 'BIRC6',\n",
       " 'ENSG00000049323': 'LTBP1',\n",
       " 'ENSG00000152689': 'RASGRP3',\n",
       " 'ENSG00000150938': 'CRIM1',\n",
       " 'ENSG00000268688': 'AC007382.1',\n",
       " 'ENSG00000115808': 'STRN',\n",
       " 'ENSG00000055332': 'EIF2AK2',\n",
       " 'ENSG00000163171': 'CDC42EP3',\n",
       " 'ENSG00000115875': 'SRSF7',\n",
       " 'ENSG00000011566': 'MAP4K3',\n",
       " 'ENSG00000231312': 'AC007246.3',\n",
       " 'ENSG00000152154': 'TMEM178A',\n",
       " 'ENSG00000162878': 'PKDCC',\n",
       " 'ENSG00000224875': 'AC083949.1',\n",
       " 'ENSG00000143924': 'EML4',\n",
       " 'ENSG00000152518': 'ZFP36L2',\n",
       " 'ENSG00000234936': 'AC010883.5',\n",
       " 'ENSG00000152527': 'PLEKHH2',\n",
       " 'ENSG00000138095': 'LRPPRC',\n",
       " 'ENSG00000116016': 'EPAS1',\n",
       " 'ENSG00000180398': 'MCFD2',\n",
       " 'ENSG00000143933': 'CALM2',\n",
       " 'ENSG00000116062': 'MSH6',\n",
       " 'ENSG00000170802': 'FOXN2',\n",
       " 'ENSG00000119737': 'GPR75',\n",
       " 'ENSG00000068878': 'PSME4',\n",
       " 'ENSG00000170634': 'ACYP2',\n",
       " 'ENSG00000115306': 'SPTBN1',\n",
       " 'ENSG00000115310': 'RTN4',\n",
       " 'ENSG00000162994': 'CLHC1',\n",
       " 'ENSG00000143947': 'RPS27A',\n",
       " 'ENSG00000115355': 'CCDC88A',\n",
       " 'ENSG00000162924': 'REL',\n",
       " 'ENSG00000115464': 'USP34',\n",
       " 'ENSG00000270820': 'RP11-355B11.2',\n",
       " 'ENSG00000082898': 'XPO1',\n",
       " 'ENSG00000115504': 'EHBP1',\n",
       " 'ENSG00000231609': 'AC009501.4',\n",
       " 'ENSG00000197329': 'PELI1',\n",
       " 'ENSG00000223935': 'AC008074.3',\n",
       " 'ENSG00000260101': 'RP11-568N6.1',\n",
       " 'ENSG00000119844': 'AFTPH',\n",
       " 'ENSG00000115902': 'SLC1A4',\n",
       " 'ENSG00000138071': 'ACTR2',\n",
       " 'ENSG00000143995': 'MEIS1',\n",
       " 'ENSG00000143971': 'ETAA1',\n",
       " 'ENSG00000273064': 'RP11-474G23.3',\n",
       " 'ENSG00000119865': 'CNRIP1',\n",
       " 'ENSG00000115956': 'PLEK',\n",
       " 'ENSG00000163219': 'ARHGAP25',\n",
       " 'ENSG00000169604': 'ANTXR1',\n",
       " 'ENSG00000198380': 'GFPT1',\n",
       " 'ENSG00000188971': 'RP11-427H3.3',\n",
       " 'ENSG00000115977': 'AAK1',\n",
       " 'ENSG00000196975': 'ANXA4',\n",
       " 'ENSG00000059728': 'MXD1',\n",
       " 'ENSG00000116005': 'PCYOX1',\n",
       " 'ENSG00000228384': 'AC007040.6',\n",
       " 'ENSG00000272735': 'RP11-467P9.1',\n",
       " 'ENSG00000075292': 'ZNF638',\n",
       " 'ENSG00000273245': 'RP11-434P11.2',\n",
       " 'ENSG00000163017': 'ACTG2',\n",
       " 'ENSG00000255989': 'RP11-711M9.1',\n",
       " 'ENSG00000163170': 'BOLA3',\n",
       " 'ENSG00000065911': 'MTHFD2',\n",
       " 'ENSG00000159239': 'C2orf81',\n",
       " 'ENSG00000135622': 'SEMA4F',\n",
       " 'ENSG00000270462': 'RP11-342K6.3',\n",
       " 'ENSG00000066032': 'CTNNA2',\n",
       " 'ENSG00000034510': 'TMSB10',\n",
       " 'ENSG00000152291': 'TGOLN2',\n",
       " 'ENSG00000042493': 'CAPG',\n",
       " 'ENSG00000168906': 'MAT2A',\n",
       " 'ENSG00000168899': 'VAMP5',\n",
       " 'ENSG00000115525': 'ST3GAL5',\n",
       " 'ENSG00000272564': 'RP11-548P2.2',\n",
       " 'ENSG00000068615': 'REEP1',\n",
       " 'ENSG00000115548': 'KDM3A',\n",
       " 'ENSG00000239305': 'RNF103',\n",
       " 'ENSG00000222041': 'LINC00152',\n",
       " 'ENSG00000273445': 'RP11-1399P15.1',\n",
       " 'ENSG00000172086': 'KRCC1',\n",
       " 'ENSG00000174501': 'ANKRD36C',\n",
       " 'ENSG00000235480': 'RP11-363D14.1',\n",
       " 'ENSG00000196843': 'ARID5A',\n",
       " 'ENSG00000273265': 'RP11-353K11.1',\n",
       " 'ENSG00000168754': 'FAM178B',\n",
       " 'ENSG00000135976': 'ANKRD36',\n",
       " 'ENSG00000230606': 'AC159540.1',\n",
       " 'ENSG00000075568': 'TMEM131',\n",
       " 'ENSG00000071073': 'MGAT4A',\n",
       " 'ENSG00000273045': 'C2ORF15',\n",
       " 'ENSG00000185674': 'LYG2',\n",
       " 'ENSG00000144214': 'LYG1',\n",
       " 'ENSG00000115604': 'IL18R1',\n",
       " 'ENSG00000272861': 'RP11-332H14.1',\n",
       " 'ENSG00000233339': 'AC018878.3',\n",
       " 'ENSG00000198075': 'SULT1C4',\n",
       " 'ENSG00000135968': 'GCC2',\n",
       " 'ENSG00000169756': 'LIMS1',\n",
       " 'ENSG00000153201': 'RANBP2',\n",
       " 'ENSG00000257207': 'LIMS3',\n",
       " 'ENSG00000169679': 'BUB1',\n",
       " 'ENSG00000172965': 'MIR4435-1HG',\n",
       " 'ENSG00000271590': 'RP11-181E10.3',\n",
       " 'ENSG00000270190': 'RP11-803D5.4',\n",
       " 'ENSG00000153214': 'TMEM87B',\n",
       " 'ENSG00000188177': 'ZC3H6',\n",
       " 'ENSG00000228251': 'AC012442.6',\n",
       " 'ENSG00000144136': 'SLC20A1',\n",
       " 'ENSG00000144130': 'NT5DC4',\n",
       " 'ENSG00000169607': 'CKAP2L',\n",
       " 'ENSG00000136689': 'IL1RN',\n",
       " 'ENSG00000270019': 'RP11-141B14.1',\n",
       " 'ENSG00000115091': 'ACTR3',\n",
       " 'ENSG00000234199': 'AC010982.1',\n",
       " 'ENSG00000155368': 'DBI',\n",
       " 'ENSG00000144118': 'RALB',\n",
       " 'ENSG00000136732': 'GYPC',\n",
       " 'ENSG00000115718': 'PROC',\n",
       " 'ENSG00000163166': 'IWS1',\n",
       " 'ENSG00000136731': 'UGGT1',\n",
       " 'ENSG00000196604': 'POTEF',\n",
       " 'ENSG00000152076': 'CCDC74B',\n",
       " 'ENSG00000136710': 'CCDC115',\n",
       " 'ENSG00000115762': 'PLEKHB2',\n",
       " 'ENSG00000188219': 'POTEE',\n",
       " 'ENSG00000178206': 'CYP4F31P',\n",
       " 'ENSG00000163040': 'CCDC74A',\n",
       " 'ENSG00000224559': 'LINC01087',\n",
       " 'ENSG00000197927': 'C2orf27A',\n",
       " 'ENSG00000152127': 'MGAT5',\n",
       " 'ENSG00000082258': 'CCNT2',\n",
       " 'ENSG00000115839': 'RAB3GAP1',\n",
       " 'ENSG00000226806': 'AC011893.3',\n",
       " 'ENSG00000115866': 'DARS',\n",
       " 'ENSG00000150540': 'HNMT',\n",
       " 'ENSG00000169554': 'ZEB2',\n",
       " 'ENSG00000121989': 'ACVR2A',\n",
       " 'ENSG00000204406': 'MBD5',\n",
       " 'ENSG00000231969': 'AC144449.1',\n",
       " 'ENSG00000183091': 'NEB',\n",
       " 'ENSG00000226383': 'AC093375.1',\n",
       " 'ENSG00000153234': 'NR4A2',\n",
       " 'ENSG00000136542': 'GALNT5',\n",
       " 'ENSG00000115165': 'CYTIP',\n",
       " 'ENSG00000234584': 'AC019186.1',\n",
       " 'ENSG00000271320': 'RP11-1152H14.1',\n",
       " 'ENSG00000115183': 'TANC1',\n",
       " 'ENSG00000123636': 'BAZ2B',\n",
       " 'ENSG00000224152': 'AC009506.1',\n",
       " 'ENSG00000136536': 'MARCH7',\n",
       " 'ENSG00000136560': 'TANK',\n",
       " 'ENSG00000115233': 'PSMD14',\n",
       " 'ENSG00000144290': 'SLC4A10',\n",
       " 'ENSG00000115267': 'IFIH1',\n",
       " 'ENSG00000115271': 'GCA',\n",
       " 'ENSG00000184611': 'KCNH7',\n",
       " 'ENSG00000169432': 'SCN9A',\n",
       " 'ENSG00000235335': 'AC016723.4',\n",
       " 'ENSG00000163072': 'NOSTRIN',\n",
       " 'ENSG00000152253': 'SPC25',\n",
       " 'ENSG00000073734': 'ABCB11',\n",
       " 'ENSG00000073737': 'DHRS9',\n",
       " 'ENSG00000239474': 'KLHL41',\n",
       " 'ENSG00000154479': 'CCDC173',\n",
       " 'ENSG00000231898': 'AC012594.1',\n",
       " 'ENSG00000239467': 'AC007405.6',\n",
       " 'ENSG00000204334': 'ERICH2',\n",
       " 'ENSG00000128683': 'GAD1',\n",
       " 'ENSG00000198586': 'TLK1',\n",
       " 'ENSG00000228389': 'AC068039.4',\n",
       " 'ENSG00000115844': 'DLX2',\n",
       " 'ENSG00000232788': 'AC078883.3',\n",
       " 'ENSG00000144354': 'CDCA7',\n",
       " 'ENSG00000271151': 'RP11-394I13.2',\n",
       " 'ENSG00000138433': 'CIR1',\n",
       " 'ENSG00000224189': 'HOXD-AS1',\n",
       " 'ENSG00000170144': 'HNRNPA3',\n",
       " 'ENSG00000116044': 'NFE2L2',\n",
       " 'ENSG00000155657': 'TTN',\n",
       " 'ENSG00000271401': 'RP11-171I2.3',\n",
       " 'ENSG00000115232': 'ITGA4',\n",
       " 'ENSG00000138434': 'SSFA2',\n",
       " 'ENSG00000077232': 'DNAJC10',\n",
       " 'ENSG00000231646': 'AC008174.3',\n",
       " 'ENSG00000138448': 'ITGAV',\n",
       " 'ENSG00000003436': 'TFPI',\n",
       " 'ENSG00000138449': 'SLC40A1',\n",
       " 'ENSG00000128694': 'OSGEPL1',\n",
       " 'ENSG00000273240': 'RP11-455J20.3',\n",
       " 'ENSG00000187699': 'C2orf88',\n",
       " 'ENSG00000151689': 'INPP1',\n",
       " 'ENSG00000151690': 'MFSD6',\n",
       " 'ENSG00000233654': 'AC093388.3',\n",
       " 'ENSG00000138386': 'NAB1',\n",
       " 'ENSG00000235852': 'AC005540.3',\n",
       " 'ENSG00000115419': 'GLS',\n",
       " 'ENSG00000115415': 'STAT1',\n",
       " 'ENSG00000173559': 'NABP1',\n",
       " 'ENSG00000235056': 'AC010983.1',\n",
       " 'ENSG00000271893': 'RP11-762E8.1',\n",
       " 'ENSG00000234919': 'AC064834.3',\n",
       " 'ENSG00000196950': 'SLC39A10',\n",
       " 'ENSG00000081320': 'STK17B',\n",
       " 'ENSG00000272211': 'RP11-347P5.1',\n",
       " 'ENSG00000229056': 'AC020571.3',\n",
       " 'ENSG00000115524': 'SF3B1',\n",
       " 'ENSG00000144381': 'HSPD1',\n",
       " 'ENSG00000115541': 'HSPE1',\n",
       " 'ENSG00000162944': 'RFTN2',\n",
       " 'ENSG00000196141': 'SPATS2L',\n",
       " 'ENSG00000163535': 'SGOL2',\n",
       " 'ENSG00000260006': 'RP11-469M7.1',\n",
       " 'ENSG00000013441': 'CLK1',\n",
       " 'ENSG00000003402': 'CFLAR',\n",
       " 'ENSG00000003400': 'CASP10',\n",
       " 'ENSG00000064012': 'CASP8',\n",
       " 'ENSG00000115993': 'TRAK2',\n",
       " 'ENSG00000138395': 'CDK15',\n",
       " 'ENSG00000273209': 'RP11-107N15.1',\n",
       " 'ENSG00000155760': 'FZD7',\n",
       " 'ENSG00000204217': 'BMPR2',\n",
       " 'ENSG00000138380': 'CARF',\n",
       " 'ENSG00000138443': 'ABI2',\n",
       " 'ENSG00000256458': 'RP11-363J17.1',\n",
       " 'ENSG00000227946': 'AC007383.3',\n",
       " 'ENSG00000204186': 'ZDBF2',\n",
       " 'ENSG00000118260': 'CREB1',\n",
       " 'ENSG00000244567': 'AC096772.6',\n",
       " 'ENSG00000138413': 'IDH1',\n",
       " 'ENSG00000144445': 'KANSL1L',\n",
       " 'ENSG00000229127': 'AC007038.7',\n",
       " 'ENSG00000030419': 'IKZF2',\n",
       " 'ENSG00000229267': 'AC072062.1',\n",
       " 'ENSG00000163449': 'TMEM169',\n",
       " 'ENSG00000079246': 'XRCC5',\n",
       " 'ENSG00000197756': 'RPL37A',\n",
       " 'ENSG00000115457': 'IGFBP2',\n",
       " 'ENSG00000273361': 'RP11-378A13.2',\n",
       " 'ENSG00000135913': 'USP37',\n",
       " 'ENSG00000135929': 'CYP27A1',\n",
       " 'ENSG00000171450': 'CDK5R2',\n",
       " 'ENSG00000163497': 'FEV',\n",
       " 'ENSG00000181378': 'CCDC108',\n",
       " 'ENSG00000115657': 'ABCB6',\n",
       " 'ENSG00000135924': 'DNAJB2',\n",
       " 'ENSG00000144591': 'GMPPA',\n",
       " 'ENSG00000072182': 'ASIC4',\n",
       " 'ENSG00000268896': 'RP11-256I23.1',\n",
       " 'ENSG00000123999': 'INHA',\n",
       " 'ENSG00000114923': 'SLC4A3',\n",
       " 'ENSG00000116120': 'FARSB',\n",
       " 'ENSG00000123983': 'ACSL3',\n",
       " 'ENSG00000152049': 'KCNE4',\n",
       " 'ENSG00000085449': 'WDFY1',\n",
       " 'ENSG00000169047': 'IRS1',\n",
       " 'ENSG00000236432': 'AC097662.2',\n",
       " 'ENSG00000153827': 'TRIP12',\n",
       " 'ENSG00000135899': 'SP110',\n",
       " 'ENSG00000067066': 'SP100',\n",
       " 'ENSG00000226125': 'AC098823.3',\n",
       " 'ENSG00000135916': 'ITM2C',\n",
       " 'ENSG00000135898': 'GPR55',\n",
       " 'ENSG00000115053': 'NCL',\n",
       " 'ENSG00000187514': 'PTMA',\n",
       " 'ENSG00000221944': 'TIGD1',\n",
       " 'ENSG00000115474': 'KCNJ13',\n",
       " 'ENSG00000182600': 'C2orf82',\n",
       " 'ENSG00000168918': 'INPP5D',\n",
       " 'ENSG00000130561': 'SAG',\n",
       " 'ENSG00000123485': 'HJURP',\n",
       " 'ENSG00000232893': 'AC019068.2',\n",
       " 'ENSG00000227252': 'AC105760.2',\n",
       " 'ENSG00000234949': 'AC104667.3',\n",
       " 'ENSG00000168427': 'KLHL30',\n",
       " 'ENSG00000144485': 'HES6',\n",
       " 'ENSG00000132326': 'PER2',\n",
       " 'ENSG00000204104': 'TRAF3IP1',\n",
       " 'ENSG00000063660': 'GPC1',\n",
       " 'ENSG00000260942': 'CAPN10-AS1',\n",
       " 'ENSG00000178623': 'GPR35',\n",
       " 'ENSG00000219159': 'AC011298.2',\n",
       " 'ENSG00000162804': 'SNED1',\n",
       " 'ENSG00000115677': 'HDLBP',\n",
       " 'ENSG00000204099': 'NEU4',\n",
       " 'ENSG00000237940': 'AC093642.3',\n",
       " 'ENSG00000091181': 'IL5RA',\n",
       " 'ENSG00000150995': 'ITPR1',\n",
       " 'ENSG00000235831': 'BHLHE40-AS1',\n",
       " 'ENSG00000134107': 'BHLHE40',\n",
       " 'ENSG00000180914': 'OXTR',\n",
       " 'ENSG00000070950': 'RAD18',\n",
       " 'ENSG00000196220': 'SRGAP3',\n",
       " 'ENSG00000206573': 'SETD5-AS1',\n",
       " 'ENSG00000168137': 'SETD5',\n",
       " 'ENSG00000250151': 'ARPC4-TTLL3',\n",
       " 'ENSG00000214021': 'TTLL3',\n",
       " 'ENSG00000163703': 'CRELD1',\n",
       " 'ENSG00000163704': 'PRRT3',\n",
       " 'ENSG00000269982': 'RP11-1020A11.2',\n",
       " 'ENSG00000206567': 'AC022007.5',\n",
       " 'ENSG00000134070': 'IRAK2',\n",
       " 'ENSG00000157017': 'GHRL',\n",
       " 'ENSG00000144560': 'VGLL4',\n",
       " 'ENSG00000144713': 'RPL32',\n",
       " 'ENSG00000132182': 'NUP210',\n",
       " 'ENSG00000163517': 'HDAC11',\n",
       " 'ENSG00000163520': 'FBLN2',\n",
       " 'ENSG00000228242': 'AC093495.4',\n",
       " 'ENSG00000131389': 'SLC6A6',\n",
       " 'ENSG00000225733': 'FGD5-AS1',\n",
       " 'ENSG00000224660': 'SH3BP5-AS1',\n",
       " 'ENSG00000131370': 'SH3BP5',\n",
       " 'ENSG00000144597': 'EAF1',\n",
       " 'ENSG00000206560': 'ANKRD28',\n",
       " 'ENSG00000131378': 'RFTN1',\n",
       " 'ENSG00000092345': 'DAZL',\n",
       " 'ENSG00000131374': 'TBC1D5',\n",
       " 'ENSG00000182568': 'SATB1',\n",
       " 'ENSG00000183960': 'KCNH8',\n",
       " 'ENSG00000183977': 'PP2D1',\n",
       " 'ENSG00000129810': 'SGOL1',\n",
       " 'ENSG00000223791': 'UBE2E1-AS1',\n",
       " 'ENSG00000224074': 'LINC00691',\n",
       " 'ENSG00000151090': 'THRB',\n",
       " 'ENSG00000272554': 'RP11-317L10.1',\n",
       " ...}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "map_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "e783fc8d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>You are an expert in single-cell biology and f...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>You are a domain expert in functional genomics...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>You are assisting with the interpretation of p...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>You are an expert in regulatory genomics. Cons...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>You are a specialist in gene perturbation expe...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2575</th>\n",
       "      <td>You are an expert in gene expression analysis....</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2576</th>\n",
       "      <td>You are a genomics expert evaluating perturbat...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2577</th>\n",
       "      <td>You are an expert analyst of perturbation data...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2578</th>\n",
       "      <td>You are a functional genomics specialist. In K...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2579</th>\n",
       "      <td>You are an expert in gene regulation studies. ...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2580 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               question answer\n",
       "0     You are an expert in single-cell biology and f...    Yes\n",
       "1     You are a domain expert in functional genomics...    Yes\n",
       "2     You are assisting with the interpretation of p...    Yes\n",
       "3     You are an expert in regulatory genomics. Cons...    Yes\n",
       "4     You are a specialist in gene perturbation expe...    Yes\n",
       "...                                                 ...    ...\n",
       "2575  You are an expert in gene expression analysis....    Yes\n",
       "2576  You are a genomics expert evaluating perturbat...    Yes\n",
       "2577  You are an expert analyst of perturbation data...    Yes\n",
       "2578  You are a functional genomics specialist. In K...    Yes\n",
       "2579  You are an expert in gene regulation studies. ...    Yes\n",
       "\n",
       "[2580 rows x 2 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "607820d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "que_new = []\n",
    "for item in df['question'].values:\n",
    "    for j in map_dict.keys():\n",
    "        if j in item:\n",
    "            item_new = item.replace(j, map_dict[j])\n",
    "            que_new.append(item_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "c2cf4e97",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2580"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(que_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "a0a01332",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['question_new'] = que_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "dd692369",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.model_selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "844cd2fb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>question_new</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>You are an expert in single-cell biology and f...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are an expert in single-cell biology and f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>You are a domain expert in functional genomics...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are a domain expert in functional genomics...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>You are assisting with the interpretation of p...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are assisting with the interpretation of p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>You are an expert in regulatory genomics. Cons...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are an expert in regulatory genomics. Cons...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>You are a specialist in gene perturbation expe...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are a specialist in gene perturbation expe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2575</th>\n",
       "      <td>You are an expert in gene expression analysis....</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are an expert in gene expression analysis....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2576</th>\n",
       "      <td>You are a genomics expert evaluating perturbat...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are a genomics expert evaluating perturbat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2577</th>\n",
       "      <td>You are an expert analyst of perturbation data...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are an expert analyst of perturbation data...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2578</th>\n",
       "      <td>You are a functional genomics specialist. In K...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are a functional genomics specialist. In K...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2579</th>\n",
       "      <td>You are an expert in gene regulation studies. ...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are an expert in gene regulation studies. ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2580 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               question answer  \\\n",
       "0     You are an expert in single-cell biology and f...    Yes   \n",
       "1     You are a domain expert in functional genomics...    Yes   \n",
       "2     You are assisting with the interpretation of p...    Yes   \n",
       "3     You are an expert in regulatory genomics. Cons...    Yes   \n",
       "4     You are a specialist in gene perturbation expe...    Yes   \n",
       "...                                                 ...    ...   \n",
       "2575  You are an expert in gene expression analysis....    Yes   \n",
       "2576  You are a genomics expert evaluating perturbat...    Yes   \n",
       "2577  You are an expert analyst of perturbation data...    Yes   \n",
       "2578  You are a functional genomics specialist. In K...    Yes   \n",
       "2579  You are an expert in gene regulation studies. ...    Yes   \n",
       "\n",
       "                                           question_new  \n",
       "0     You are an expert in single-cell biology and f...  \n",
       "1     You are a domain expert in functional genomics...  \n",
       "2     You are assisting with the interpretation of p...  \n",
       "3     You are an expert in regulatory genomics. Cons...  \n",
       "4     You are a specialist in gene perturbation expe...  \n",
       "...                                                 ...  \n",
       "2575  You are an expert in gene expression analysis....  \n",
       "2576  You are a genomics expert evaluating perturbat...  \n",
       "2577  You are an expert analyst of perturbation data...  \n",
       "2578  You are a functional genomics specialist. In K...  \n",
       "2579  You are an expert in gene regulation studies. ...  \n",
       "\n",
       "[2580 rows x 3 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "f5e44c1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_id, test_id = sklearn.model_selection.train_test_split(list(df.index), test_size=2500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "4521c224",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "answer\n",
       "Yes    78\n",
       "No      2\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[train_id]['answer'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "e7232a41",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[train_id].to_csv(\"adamson_train.csv\")\n",
    "df.loc[test_id].to_csv(\"adamson_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "494a152f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2500"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df.loc[test_id])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "260eb2b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/adamson_train.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "577bcf5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_d = df.loc[train_id]\n",
    "train_d['_id'] = list(train_d.index)\n",
    "\n",
    "test_d = df.loc[test_id]\n",
    "test_d['_id'] = list(test_d.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "e666ea86",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_d.to_json(\"./adamson_train.json\", orient='records', indent=2)\n",
    "test_d.to_json(\"./adamson_test.json\", orient='records', indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08f544f6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4c2a6fa7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/PyPDF2/__init__.py:21: DeprecationWarning: PyPDF2 is deprecated. Please move to the pypdf library instead.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from evoagentx.agents.agent_manager import AgentManager\n",
    "from evoagentx.benchmark import HotPotQA\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.core.logging import logger\n",
    "from evoagentx.evaluators import Evaluator\n",
    "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
    "from evoagentx.optimizers import TextGradOptimizer\n",
    "from evoagentx.prompts import StringTemplate\n",
    "from evoagentx.workflow import SequentialWorkFlowGraph\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from evoagentx.agents.agent_manager import AgentManager\n",
    "from evoagentx.benchmark import MBPP\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.core.logging import logger\n",
    "from evoagentx.evaluators import Evaluator\n",
    "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
    "from evoagentx.optimizers import TextGradOptimizer\n",
    "from evoagentx.prompts import StringTemplate\n",
    "from evoagentx.workflow import SequentialWorkFlowGraph\n",
    "\n",
    "from evoagentx.models import OpenAILLMConfig, OpenAILLM\n",
    "from evoagentx.workflow import SEWWorkFlowGraph, STRUCTUREWorkFlowGraph\n",
    "from evoagentx.agents import AgentManager\n",
    "from evoagentx.benchmark import HumanEval,AFlowMBPP\n",
    "from evoagentx.evaluators import Evaluator \n",
    "from evoagentx.optimizers import SEWOptimizer, STRUCTUREOptimizer\n",
    "from evoagentx.optimizers.structure_optimizer import STRUCTUREWorkFlowScheme\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "\n",
    "from evoagentx.models import OpenAILLMConfig, OpenAILLM,AzureOpenAIConfig,LiteLLMConfig,LiteLLM\n",
    "from evoagentx.workflow import SEWWorkFlowGraph \n",
    "from evoagentx.agents import AgentManager\n",
    "from evoagentx.benchmark import MBPPPLUS, AFlowMBPPPLUS\n",
    "from evoagentx.evaluators import Evaluator \n",
    "from evoagentx.optimizers import SEWOptimizer \n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.benchmark import HumanEvalPLUS\n",
    "from evoagentx.benchmark import SciCode\n",
    "from evoagentx.benchmark import PertQA\n",
    "from copy import deepcopy\n",
    "\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()\n",
    "\n",
    "class PertQASplits(PertQA):\n",
    "    def _load_data(self):\n",
    "        # load the original test data \n",
    "        super()._load_data(pertdata = 'adamson')\n",
    "        # split the data into train, dev and test\n",
    "        import numpy as np \n",
    "        np.random.seed(42)\n",
    "        permutation = np.random.permutation(len(self._dev_data))\n",
    "        full_test_data = self._dev_data \n",
    "        # randomly select 10 samples for train, 40 for dev, and 100 for test\n",
    "        self._train_data = [full_test_data[idx] for idx in permutation[:50]]\n",
    "        self._dev_data = [full_test_data[idx] for idx in permutation[:50]]\n",
    "        self._fulldata = full_test_data\n",
    "\n",
    "\n",
    "def collate_func(example: dict) -> dict:\n",
    "    problem = \"Question: {}\\n\\nAnswer:\".format(example[\"question\"])\n",
    "    return {\"problem\": problem}\n",
    "\n",
    "\n",
    "hotpotqa_graph_data = {\n",
    "    \"goal\": \"Answer the question based on the context. The answer should be a direct response to the question, without including explanations or reasoning.\",\n",
    "    \"tasks\": [\n",
    "        {\n",
    "            \"name\": \"answer_generate\",\n",
    "            \"description\": \"Answer the question based on the context.\",\n",
    "            \"inputs\": [\n",
    "                {\"name\": \"problem\", \"type\": \"str\", \"required\": True, \"description\": \"The problem to solve.\"}\n",
    "            ],\n",
    "            \"outputs\": [\n",
    "                {\"name\": \"answer\", \"type\": \"str\", \"required\": True, \"description\": \"The answer to the problem.\"}\n",
    "            ],\n",
    "            \"prompt_template\": StringTemplate(instruction=\"Think step by step to answer the question. You should explain your thinking process in the 'thought' field, and provide the final answer in the 'answer' field.\\nFormat your output in xml format, such as <thought>xxx</thought> and <answer>xxx</answer>.\"),\n",
    "            \"parse_mode\": \"xml\"\n",
    "        }\n",
    "    ] \n",
    "}\n",
    "api_key = \"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\"\n",
    "# Function to encode the image\n",
    "import os\n",
    "os.environ[\"OPENAI_API_KEY\"] = api_key\n",
    "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n",
    "\n",
    "\n",
    "llm_config = OpenAILLMConfig(model=\"gpt-4o-mini-2024-07-18\", openai_key=OPENAI_API_KEY, top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
    "llm = OpenAILLM(config=llm_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ad0efa03",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "evoagentx.optimizers.sew_optimizer.SEWOptimizer"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SEWOptimizer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ad4b2024",
   "metadata": {},
   "outputs": [],
   "source": [
    "# difficult easy "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c95059f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from evoagentx.benchmark import HotPotQA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "84efabfa",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 16:28:53.088\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/adamson_train.json ...\u001b[0m\n",
      "\u001b[32m2025-12-27 16:28:53.088\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/adamson_train.json ...\u001b[0m\n",
      "\u001b[32m2025-12-27 16:28:53.089\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/adamson_test.json ...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# llm_config = OpenAILLMConfig(model=\"gpt-4o-mini-2024-07-18\", openai_key=OPENAI_API_KEY, top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
    "# llm = OpenAILLM(config=llm_config)\n",
    "\n",
    "# obtain SEW workflow \n",
    "sew_graph = SEWWorkFlowGraph.from_dict(hotpotqa_graph_data)\n",
    "agent_manager = AgentManager()\n",
    "agent_manager.add_agents_from_workflow(sew_graph, llm.config)\n",
    "\n",
    "benchmark = PertQA(pertdata='adamson')\n",
    "\n",
    "# obtain Evaluator\n",
    "evaluator = Evaluator(llm=llm, agent_manager=agent_manager, collate_func=collate_func, num_workers=20, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d2bba683",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import json\n",
    "# # with open(\"../../MaAS/maas/ext/maas/data/humaneval_train.jsonl\", 'w') as f:\n",
    "# #     json.dump(humaneval._dev_data, f, indent=2) # indent=4 makes the JSON output more readable\n",
    "\n",
    "\n",
    "# # with open(\"../../MaAS/maas/ext/maas/data/humaneval_test.jsonl\", 'w') as f:\n",
    "# #     json.dump(humaneval._test_data, f, indent=2) # indent=4 makes the JSON output more readable\n",
    "\n",
    "# with open(\"../../MaAS/maas/ext/maas/data/humaneval_train.jsonl\", 'w') as f:\n",
    "#     for obj in humaneval._dev_data:\n",
    "#         json_line = json.dumps(obj)\n",
    "#         f.write(json_line + '\\n')\n",
    "        \n",
    "# with open(\"../../MaAS/maas/ext/maas/data/humaneval_test.jsonl\", 'w') as f:\n",
    "#     for obj in humaneval._test_data:\n",
    "#         json_line = json.dumps(obj)\n",
    "#         f.write(json_line + '\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8598151b",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sew_graph.to_dict()['nodes'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b1f7fc18",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sew_graph.edges)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "33859fa8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sew_graph.edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3c048529",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 5/2500 [00:02<18:17,  2.27it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 7/2500 [00:03<14:10,  2.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 14/2500 [00:03<05:42,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 16/2500 [00:03<05:42,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 18/2500 [00:03<05:04,  8.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 20/2500 [00:04<07:49,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 22/2500 [00:05<09:23,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 23/2500 [00:05<09:18,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 24/2500 [00:05<09:45,  4.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 27/2500 [00:06<08:23,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|▏         | 34/2500 [00:06<04:05, 10.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 36/2500 [00:06<04:36,  8.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 38/2500 [00:07<04:23,  9.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 41/2500 [00:08<07:15,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 42/2500 [00:08<08:33,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 45/2500 [00:09<09:03,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 49/2500 [00:09<07:44,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 53/2500 [00:09<05:03,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 56/2500 [00:10<04:16,  9.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 58/2500 [00:10<05:18,  7.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 59/2500 [00:11<07:12,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 60/2500 [00:11<07:24,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 62/2500 [00:11<06:57,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 67/2500 [00:12<05:35,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 70/2500 [00:12<06:42,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 74/2500 [00:13<05:07,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 76/2500 [00:13<05:10,  7.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 79/2500 [00:13<05:14,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 81/2500 [00:14<04:57,  8.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 82/2500 [00:14<05:58,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 84/2500 [00:15<09:59,  4.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 87/2500 [00:15<05:41,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 90/2500 [00:15<06:13,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 92/2500 [00:16<06:09,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 94/2500 [00:16<05:55,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 96/2500 [00:17<07:23,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 99/2500 [00:17<06:59,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 103/2500 [00:17<05:02,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 104/2500 [00:18<05:27,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 108/2500 [00:18<03:52, 10.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 110/2500 [00:19<06:56,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 112/2500 [00:19<05:25,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 114/2500 [00:19<08:04,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 116/2500 [00:20<06:51,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 118/2500 [00:20<06:01,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 120/2500 [00:20<06:34,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 122/2500 [00:21<06:15,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 124/2500 [00:21<07:33,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 126/2500 [00:21<05:55,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 128/2500 [00:22<06:21,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 132/2500 [00:22<04:03,  9.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 135/2500 [00:22<04:14,  9.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 136/2500 [00:23<07:34,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 138/2500 [00:23<07:33,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 140/2500 [00:23<06:27,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 141/2500 [00:24<06:19,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 143/2500 [00:24<06:43,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 145/2500 [00:24<06:27,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 150/2500 [00:24<03:23, 11.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 154/2500 [00:25<03:14, 12.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 156/2500 [00:26<07:50,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 159/2500 [00:26<07:11,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 163/2500 [00:27<06:10,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 164/2500 [00:27<05:50,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 170/2500 [00:27<03:38, 10.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 172/2500 [00:28<05:10,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 174/2500 [00:28<05:46,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 175/2500 [00:28<06:24,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 177/2500 [00:29<08:58,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 181/2500 [00:29<04:25,  8.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 183/2500 [00:30<06:15,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 186/2500 [00:30<06:17,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 188/2500 [00:31<05:21,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 192/2500 [00:31<06:23,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 196/2500 [00:32<05:48,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 199/2500 [00:32<05:22,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 202/2500 [00:33<05:47,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 203/2500 [00:33<07:03,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 204/2500 [00:33<08:00,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 208/2500 [00:34<05:26,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 210/2500 [00:34<04:45,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 212/2500 [00:34<04:39,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 213/2500 [00:35<06:02,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▊         | 216/2500 [00:35<05:58,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 218/2500 [00:35<05:03,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 221/2500 [00:36<06:43,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 223/2500 [00:36<05:41,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 225/2500 [00:37<07:37,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 228/2500 [00:37<06:56,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 232/2500 [00:38<05:38,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 235/2500 [00:38<04:30,  8.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 237/2500 [00:38<03:45, 10.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 239/2500 [00:39<06:25,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 240/2500 [00:39<06:40,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 243/2500 [00:40<07:12,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 246/2500 [00:40<06:52,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 248/2500 [00:41<07:45,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 254/2500 [00:41<03:46,  9.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 258/2500 [00:41<03:01, 12.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 260/2500 [00:42<07:23,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 262/2500 [00:43<08:56,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 263/2500 [00:44<10:58,  3.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 270/2500 [00:44<04:55,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 272/2500 [00:44<05:27,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 274/2500 [00:45<05:50,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 277/2500 [00:45<04:58,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 280/2500 [00:46<05:50,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 281/2500 [00:46<08:54,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 287/2500 [00:47<04:44,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 289/2500 [00:47<05:09,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 290/2500 [00:47<04:55,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 291/2500 [00:48<06:44,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 295/2500 [00:48<04:25,  8.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 296/2500 [00:48<04:56,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 297/2500 [00:48<05:52,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 298/2500 [00:49<08:07,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 302/2500 [00:49<06:20,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 305/2500 [00:50<05:49,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 310/2500 [00:50<04:09,  8.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 312/2500 [00:51<04:38,  7.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 314/2500 [00:51<04:36,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 316/2500 [00:51<05:02,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 317/2500 [00:52<08:49,  4.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 320/2500 [00:52<06:42,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 321/2500 [00:53<07:47,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 322/2500 [00:53<09:16,  3.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 324/2500 [00:53<08:14,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 329/2500 [00:54<04:37,  7.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 332/2500 [00:54<04:23,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 333/2500 [00:55<06:51,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 336/2500 [00:55<05:53,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 337/2500 [00:55<06:25,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 339/2500 [00:56<07:34,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 343/2500 [00:57<07:53,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 345/2500 [00:57<06:29,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 348/2500 [00:58<05:31,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 351/2500 [00:58<05:17,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 353/2500 [00:58<04:59,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 355/2500 [00:59<04:47,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 359/2500 [01:00<06:24,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 361/2500 [01:00<06:33,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 363/2500 [01:00<07:09,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 364/2500 [01:01<06:22,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 366/2500 [01:01<05:41,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 368/2500 [01:01<06:09,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 370/2500 [01:01<05:42,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 372/2500 [01:02<07:02,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 374/2500 [01:02<06:52,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 375/2500 [01:03<10:01,  3.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 377/2500 [01:03<07:54,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 378/2500 [01:04<08:18,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 380/2500 [01:04<07:34,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 382/2500 [01:04<06:27,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 385/2500 [01:05<05:27,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 387/2500 [01:05<05:27,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 390/2500 [01:05<04:24,  7.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 391/2500 [01:06<06:50,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 393/2500 [01:06<07:16,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 395/2500 [01:06<06:20,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 397/2500 [01:07<06:27,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 398/2500 [01:07<07:39,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 401/2500 [01:07<05:22,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 402/2500 [01:08<06:09,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 403/2500 [01:08<07:21,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 404/2500 [01:08<08:46,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 405/2500 [01:09<10:30,  3.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 406/2500 [01:09<10:14,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 409/2500 [01:09<06:38,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 413/2500 [01:10<05:05,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 414/2500 [01:10<05:41,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 418/2500 [01:11<04:08,  8.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 422/2500 [01:11<03:56,  8.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 424/2500 [01:12<07:58,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 426/2500 [01:13<07:58,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 427/2500 [01:13<07:26,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 431/2500 [01:13<06:00,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 434/2500 [01:14<05:51,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 436/2500 [01:14<04:40,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 439/2500 [01:14<04:55,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 443/2500 [01:15<05:25,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 444/2500 [01:15<06:13,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 445/2500 [01:16<06:44,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 448/2500 [01:16<06:27,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 450/2500 [01:17<05:46,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 454/2500 [01:17<04:12,  8.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 456/2500 [01:18<06:13,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 457/2500 [01:18<06:48,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 460/2500 [01:18<05:22,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 462/2500 [01:19<05:28,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 463/2500 [01:19<05:22,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 465/2500 [01:19<07:03,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 468/2500 [01:19<04:14,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 470/2500 [01:20<04:32,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 471/2500 [01:20<07:11,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 474/2500 [01:21<07:13,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 477/2500 [01:21<04:45,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 479/2500 [01:21<03:50,  8.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 481/2500 [01:22<06:41,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 484/2500 [01:22<05:03,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 485/2500 [01:23<06:03,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 487/2500 [01:23<06:18,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 491/2500 [01:24<05:14,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 492/2500 [01:24<06:59,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 493/2500 [01:24<07:26,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 495/2500 [01:25<05:59,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 497/2500 [01:25<05:08,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 501/2500 [01:25<04:33,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 502/2500 [01:25<04:20,  7.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 503/2500 [01:26<05:42,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 504/2500 [01:26<07:19,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 506/2500 [01:27<07:07,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 507/2500 [01:27<09:32,  3.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 508/2500 [01:28<12:00,  2.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 514/2500 [01:28<05:14,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 517/2500 [01:29<05:00,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 519/2500 [01:29<04:45,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 522/2500 [01:29<04:23,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 527/2500 [01:30<04:08,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 529/2500 [01:31<07:30,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 531/2500 [01:31<06:15,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 532/2500 [01:32<09:22,  3.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 535/2500 [01:32<07:15,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 536/2500 [01:32<06:38,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 540/2500 [01:33<04:23,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 542/2500 [01:33<03:49,  8.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 543/2500 [01:33<05:15,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 545/2500 [01:33<04:45,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 547/2500 [01:34<05:27,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 548/2500 [01:35<08:41,  3.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 551/2500 [01:35<07:22,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 555/2500 [01:36<04:16,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 557/2500 [01:36<05:21,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 561/2500 [01:36<03:52,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 563/2500 [01:37<04:58,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 565/2500 [01:37<04:59,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 567/2500 [01:37<04:58,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 568/2500 [01:38<09:07,  3.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 570/2500 [01:39<07:46,  4.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 572/2500 [01:39<05:41,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 577/2500 [01:39<03:28,  9.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 580/2500 [01:40<04:32,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 584/2500 [01:40<03:46,  8.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 586/2500 [01:40<03:51,  8.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 589/2500 [01:41<05:37,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 591/2500 [01:41<05:14,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 592/2500 [01:42<07:36,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 593/2500 [01:42<08:35,  3.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 597/2500 [01:43<05:09,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 599/2500 [01:43<04:18,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 601/2500 [01:43<04:59,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 602/2500 [01:44<05:44,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 605/2500 [01:44<05:12,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 608/2500 [01:44<04:30,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 611/2500 [01:45<03:07, 10.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 613/2500 [01:46<07:39,  4.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 620/2500 [01:46<04:22,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 622/2500 [01:47<05:57,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 625/2500 [01:47<04:58,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 628/2500 [01:48<05:44,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 631/2500 [01:49<05:30,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 632/2500 [01:49<05:23,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 634/2500 [01:49<05:41,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 637/2500 [01:50<04:16,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 639/2500 [01:50<04:41,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 641/2500 [01:50<05:56,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 642/2500 [01:51<05:23,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 644/2500 [01:51<04:33,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 645/2500 [01:51<05:42,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 647/2500 [01:51<05:52,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 650/2500 [01:52<05:38,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 652/2500 [01:52<05:23,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 653/2500 [01:53<07:20,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 656/2500 [01:53<05:03,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 657/2500 [01:53<05:32,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 661/2500 [01:54<04:49,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 662/2500 [01:54<06:12,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 665/2500 [01:55<04:49,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 667/2500 [01:55<03:52,  7.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 670/2500 [01:55<04:25,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 672/2500 [01:56<06:50,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 675/2500 [01:57<05:29,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 677/2500 [01:57<04:08,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 681/2500 [01:57<03:45,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 683/2500 [01:58<04:21,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 686/2500 [01:58<04:41,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 689/2500 [01:58<03:36,  8.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 691/2500 [01:59<06:09,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 693/2500 [02:00<07:23,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 696/2500 [02:00<05:29,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 698/2500 [02:00<05:38,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 702/2500 [02:01<04:10,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 705/2500 [02:01<03:19,  8.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 707/2500 [02:02<06:18,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 712/2500 [02:03<04:29,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 714/2500 [02:03<05:26,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 719/2500 [02:04<04:38,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 720/2500 [02:04<05:28,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 723/2500 [02:05<04:58,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 728/2500 [02:05<04:08,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 730/2500 [02:06<05:33,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 732/2500 [02:06<05:15,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 735/2500 [02:07<05:53,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 737/2500 [02:07<04:32,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 739/2500 [02:08<04:48,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 741/2500 [02:08<04:38,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 743/2500 [02:08<04:20,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 745/2500 [02:08<03:57,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 746/2500 [02:09<09:08,  3.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 749/2500 [02:10<05:57,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 751/2500 [02:10<05:20,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 753/2500 [02:10<04:52,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 757/2500 [02:10<03:49,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 759/2500 [02:11<03:50,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 760/2500 [02:11<04:36,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 761/2500 [02:11<05:25,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 765/2500 [02:12<06:36,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 767/2500 [02:13<06:30,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 771/2500 [02:13<04:30,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 774/2500 [02:13<03:18,  8.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 776/2500 [02:14<04:54,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 780/2500 [02:15<04:53,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 781/2500 [02:15<04:58,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 785/2500 [02:16<04:20,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 786/2500 [02:16<05:22,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 789/2500 [02:17<05:58,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 791/2500 [02:17<05:15,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 794/2500 [02:17<04:49,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 796/2500 [02:18<05:40,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 799/2500 [02:18<04:26,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 801/2500 [02:19<05:09,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 803/2500 [02:19<05:07,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 805/2500 [02:20<05:06,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 807/2500 [02:20<04:25,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 810/2500 [02:20<02:46, 10.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 814/2500 [02:21<05:06,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 817/2500 [02:22<04:25,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 819/2500 [02:22<05:02,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 822/2500 [02:22<04:29,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 826/2500 [02:23<03:16,  8.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 830/2500 [02:24<03:40,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 832/2500 [02:24<03:47,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 833/2500 [02:24<04:48,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 834/2500 [02:24<05:14,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 835/2500 [02:25<07:23,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 838/2500 [02:25<04:58,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 839/2500 [02:25<05:29,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 842/2500 [02:26<04:27,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 843/2500 [02:26<04:09,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 845/2500 [02:26<04:52,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 847/2500 [02:27<03:56,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 849/2500 [02:27<04:37,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 852/2500 [02:28<05:18,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 855/2500 [02:28<04:56,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 856/2500 [02:29<06:31,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 860/2500 [02:29<04:10,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 862/2500 [02:29<04:07,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 863/2500 [02:30<05:46,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 865/2500 [02:30<05:19,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 870/2500 [02:31<04:01,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 872/2500 [02:31<04:09,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 873/2500 [02:31<04:32,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 876/2500 [02:32<03:26,  7.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 878/2500 [02:32<03:50,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 879/2500 [02:32<04:54,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 881/2500 [02:33<05:24,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 883/2500 [02:33<04:24,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 884/2500 [02:33<04:14,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 888/2500 [02:34<03:12,  8.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 889/2500 [02:34<04:30,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 892/2500 [02:34<04:12,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 895/2500 [02:35<03:32,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 897/2500 [02:35<03:36,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 898/2500 [02:36<05:10,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 899/2500 [02:36<06:36,  4.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 903/2500 [02:36<04:08,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 906/2500 [02:37<03:49,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 910/2500 [02:37<04:08,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 911/2500 [02:38<04:33,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 915/2500 [02:38<04:17,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 916/2500 [02:39<04:41,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 917/2500 [02:39<07:29,  3.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 922/2500 [02:40<04:00,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 924/2500 [02:40<03:29,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 928/2500 [02:40<02:55,  8.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 930/2500 [02:41<04:03,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 934/2500 [02:42<04:32,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 936/2500 [02:42<03:58,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 938/2500 [02:43<05:42,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 943/2500 [02:43<04:11,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 945/2500 [02:44<03:54,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 947/2500 [02:44<04:09,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 949/2500 [02:44<04:11,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 951/2500 [02:45<03:58,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 952/2500 [02:45<04:22,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 955/2500 [02:45<04:00,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 956/2500 [02:46<04:49,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 957/2500 [02:46<05:50,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 960/2500 [02:46<04:19,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 961/2500 [02:46<04:01,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 963/2500 [02:47<04:26,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 967/2500 [02:47<02:58,  8.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 970/2500 [02:48<05:35,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 972/2500 [02:48<04:26,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 974/2500 [02:49<04:09,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 975/2500 [02:49<05:09,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 980/2500 [02:50<03:12,  7.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 982/2500 [02:50<02:41,  9.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 984/2500 [02:50<03:34,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 988/2500 [02:51<03:38,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 990/2500 [02:51<03:59,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 991/2500 [02:52<04:46,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 993/2500 [02:52<04:37,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 995/2500 [02:52<04:03,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 997/2500 [02:52<03:39,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 999/2500 [02:53<05:13,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1003/2500 [02:53<03:18,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1006/2500 [02:54<02:23, 10.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1008/2500 [02:54<03:03,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1010/2500 [02:54<03:34,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1011/2500 [02:55<04:20,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1013/2500 [02:55<04:09,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1015/2500 [02:56<06:18,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1018/2500 [02:56<04:16,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1022/2500 [02:56<02:46,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1025/2500 [02:57<03:06,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1027/2500 [02:57<04:11,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1028/2500 [02:58<04:53,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1031/2500 [02:58<03:51,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1032/2500 [02:58<03:49,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1033/2500 [02:59<06:10,  3.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████▏     | 1036/2500 [02:59<04:48,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1038/2500 [03:00<04:15,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1042/2500 [03:00<03:34,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1046/2500 [03:00<02:12, 11.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1048/2500 [03:01<03:39,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1050/2500 [03:01<04:04,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1053/2500 [03:02<04:55,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1054/2500 [03:02<04:55,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1058/2500 [03:03<03:37,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1060/2500 [03:03<03:29,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1061/2500 [03:04<04:50,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1065/2500 [03:04<03:08,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1067/2500 [03:04<02:43,  8.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1070/2500 [03:05<03:24,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1071/2500 [03:05<03:30,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1074/2500 [03:05<04:11,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1075/2500 [03:06<05:19,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1078/2500 [03:06<04:03,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1081/2500 [03:07<03:10,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1083/2500 [03:07<04:46,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1085/2500 [03:07<03:31,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1089/2500 [03:07<02:07, 11.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1091/2500 [03:08<04:15,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1093/2500 [03:09<04:32,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1095/2500 [03:09<03:47,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1097/2500 [03:09<03:57,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1100/2500 [03:10<03:29,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1101/2500 [03:10<05:04,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1102/2500 [03:11<05:29,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1106/2500 [03:11<03:36,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1108/2500 [03:11<03:20,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1110/2500 [03:12<04:12,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1113/2500 [03:12<04:04,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1116/2500 [03:13<04:03,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1118/2500 [03:13<04:00,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1120/2500 [03:13<04:02,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1121/2500 [03:14<04:26,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1123/2500 [03:14<04:29,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1126/2500 [03:15<04:35,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1128/2500 [03:15<03:57,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1131/2500 [03:15<03:35,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1132/2500 [03:16<03:52,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1133/2500 [03:16<04:03,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1137/2500 [03:16<03:08,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1138/2500 [03:17<03:57,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1141/2500 [03:17<03:20,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1142/2500 [03:18<05:02,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1143/2500 [03:18<05:19,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1145/2500 [03:18<05:05,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1147/2500 [03:19<04:38,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1149/2500 [03:19<03:36,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1151/2500 [03:19<03:03,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1153/2500 [03:20<04:15,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1155/2500 [03:20<03:19,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 1158/2500 [03:20<03:50,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1160/2500 [03:21<03:31,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1162/2500 [03:21<03:14,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1165/2500 [03:22<03:51,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1167/2500 [03:22<03:37,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1168/2500 [03:22<03:52,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1171/2500 [03:23<04:52,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1172/2500 [03:23<04:25,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1174/2500 [03:23<03:42,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1176/2500 [03:24<03:48,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1179/2500 [03:24<03:03,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1183/2500 [03:24<02:20,  9.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1187/2500 [03:25<01:52, 11.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1189/2500 [03:26<04:20,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1191/2500 [03:26<03:50,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1194/2500 [03:27<05:36,  3.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1199/2500 [03:27<03:08,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1201/2500 [03:27<02:34,  8.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1203/2500 [03:28<02:57,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1205/2500 [03:28<02:54,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1208/2500 [03:29<04:24,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1211/2500 [03:29<03:43,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1212/2500 [03:30<03:34,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1213/2500 [03:30<06:20,  3.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1215/2500 [03:31<04:55,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1216/2500 [03:31<05:12,  4.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1219/2500 [03:31<03:33,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1223/2500 [03:31<02:31,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1225/2500 [03:32<02:09,  9.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1227/2500 [03:32<03:28,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1229/2500 [03:33<03:40,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1230/2500 [03:33<03:43,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1231/2500 [03:33<04:02,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1235/2500 [03:34<04:35,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1238/2500 [03:34<03:14,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1242/2500 [03:34<02:04, 10.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1244/2500 [03:35<02:12,  9.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1246/2500 [03:35<02:15,  9.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1248/2500 [03:36<05:46,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1251/2500 [03:37<04:33,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1254/2500 [03:37<03:37,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1257/2500 [03:38<03:24,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1260/2500 [03:38<02:59,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1263/2500 [03:38<02:57,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1266/2500 [03:39<01:57, 10.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1268/2500 [03:39<04:08,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1270/2500 [03:40<04:53,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1272/2500 [03:40<03:46,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1274/2500 [03:40<02:44,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1276/2500 [03:41<02:40,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1278/2500 [03:41<02:43,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1281/2500 [03:41<02:47,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 1283/2500 [03:41<02:10,  9.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 1285/2500 [03:42<02:16,  8.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1288/2500 [03:43<05:44,  3.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1290/2500 [03:44<04:14,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1292/2500 [03:44<04:22,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1294/2500 [03:44<03:20,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1297/2500 [03:44<02:47,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1299/2500 [03:45<02:47,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1304/2500 [03:45<01:47, 11.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1306/2500 [03:46<02:39,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1308/2500 [03:46<04:10,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1309/2500 [03:47<04:15,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1310/2500 [03:47<05:19,  3.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1312/2500 [03:48<05:08,  3.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1319/2500 [03:48<02:26,  8.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1321/2500 [03:48<02:27,  7.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1323/2500 [03:49<03:09,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1324/2500 [03:49<03:05,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1326/2500 [03:49<03:37,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1327/2500 [03:50<04:13,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1329/2500 [03:50<04:01,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1330/2500 [03:51<05:08,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1334/2500 [03:51<03:01,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▎    | 1338/2500 [03:51<02:38,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1339/2500 [03:52<02:52,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▎    | 1343/2500 [03:52<02:32,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1344/2500 [03:52<02:45,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1345/2500 [03:53<04:07,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1347/2500 [03:53<03:47,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1348/2500 [03:53<03:53,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1350/2500 [03:54<04:19,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1352/2500 [03:54<03:35,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1354/2500 [03:54<02:36,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1355/2500 [03:55<03:52,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1362/2500 [03:55<01:43, 10.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1364/2500 [03:55<02:25,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1366/2500 [03:56<03:11,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1368/2500 [03:56<03:24,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1370/2500 [03:57<03:57,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1372/2500 [03:57<03:37,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1377/2500 [03:58<02:18,  8.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1379/2500 [03:58<02:10,  8.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1381/2500 [03:58<01:50, 10.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1383/2500 [03:58<02:01,  9.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1385/2500 [03:59<02:20,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1388/2500 [03:59<03:05,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1390/2500 [04:00<03:22,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1393/2500 [04:00<02:54,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1397/2500 [04:01<02:43,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1399/2500 [04:01<03:34,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1401/2500 [04:02<02:37,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1402/2500 [04:02<03:09,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1403/2500 [04:02<04:35,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1407/2500 [04:03<02:56,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1410/2500 [04:03<02:54,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1413/2500 [04:04<02:56,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1414/2500 [04:04<02:44,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1417/2500 [04:04<02:54,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1419/2500 [04:05<02:18,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1422/2500 [04:05<02:25,  7.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1424/2500 [04:06<02:57,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1425/2500 [04:06<04:32,  3.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1428/2500 [04:07<03:13,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1429/2500 [04:07<03:09,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1432/2500 [04:07<02:35,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1433/2500 [04:07<03:11,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1436/2500 [04:08<03:10,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1441/2500 [04:08<02:20,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1445/2500 [04:09<02:50,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1447/2500 [04:10<02:44,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1448/2500 [04:10<04:50,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1453/2500 [04:11<03:09,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1455/2500 [04:11<02:40,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1457/2500 [04:11<02:39,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1459/2500 [04:12<02:35,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1462/2500 [04:12<02:28,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1463/2500 [04:13<03:18,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1465/2500 [04:13<03:17,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1468/2500 [04:13<03:03,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1471/2500 [04:14<02:48,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1474/2500 [04:14<02:53,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1475/2500 [04:15<03:16,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1479/2500 [04:15<02:49,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1480/2500 [04:16<03:47,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1484/2500 [04:16<02:59,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1488/2500 [04:17<01:50,  9.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1490/2500 [04:17<02:39,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1493/2500 [04:18<03:14,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1496/2500 [04:18<02:26,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1498/2500 [04:19<02:51,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1499/2500 [04:19<02:56,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1502/2500 [04:19<02:18,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1506/2500 [04:20<02:16,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1508/2500 [04:20<02:46,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1509/2500 [04:20<02:31,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1510/2500 [04:21<04:33,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1512/2500 [04:21<04:04,  4.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1516/2500 [04:22<02:40,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1521/2500 [04:22<01:51,  8.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1523/2500 [04:22<01:34, 10.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1525/2500 [04:23<02:09,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1527/2500 [04:23<02:02,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1529/2500 [04:24<02:22,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 1532/2500 [04:24<02:55,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1534/2500 [04:25<03:34,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1536/2500 [04:25<03:16,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1537/2500 [04:25<03:39,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1541/2500 [04:26<02:15,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1544/2500 [04:27<02:46,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1547/2500 [04:27<02:27,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1552/2500 [04:28<02:08,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1554/2500 [04:28<02:22,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1556/2500 [04:29<03:16,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1558/2500 [04:29<02:20,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1560/2500 [04:29<02:09,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1563/2500 [04:29<02:16,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1564/2500 [04:30<02:07,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1567/2500 [04:30<02:57,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1568/2500 [04:31<04:00,  3.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1571/2500 [04:31<02:36,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1574/2500 [04:31<02:00,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1576/2500 [04:32<02:05,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1577/2500 [04:32<02:53,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1583/2500 [04:32<01:41,  9.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1585/2500 [04:33<01:59,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1588/2500 [04:34<02:38,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1590/2500 [04:34<03:26,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1592/2500 [04:35<03:40,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1595/2500 [04:35<02:51,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1599/2500 [04:36<01:47,  8.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1602/2500 [04:36<01:24, 10.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1605/2500 [04:36<01:52,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1609/2500 [04:37<02:09,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1611/2500 [04:38<02:53,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1614/2500 [04:38<02:58,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1615/2500 [04:38<02:43,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1616/2500 [04:39<02:48,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1619/2500 [04:39<02:25,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1622/2500 [04:39<01:33,  9.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1626/2500 [04:40<01:28,  9.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1628/2500 [04:40<01:58,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1630/2500 [04:41<02:13,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1633/2500 [04:42<03:27,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1637/2500 [04:42<02:14,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1639/2500 [04:42<01:54,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1641/2500 [04:42<01:46,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1644/2500 [04:43<02:05,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1647/2500 [04:43<01:51,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1648/2500 [04:43<01:47,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1652/2500 [04:45<02:36,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1653/2500 [04:45<02:34,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1656/2500 [04:45<02:00,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1658/2500 [04:45<02:09,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1659/2500 [04:45<02:02,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1660/2500 [04:46<03:01,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1664/2500 [04:46<02:00,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1666/2500 [04:47<01:43,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1670/2500 [04:47<01:10, 11.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1672/2500 [04:48<02:50,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1674/2500 [04:48<03:11,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1677/2500 [04:49<02:31,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1678/2500 [04:49<02:36,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1681/2500 [04:49<02:20,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1682/2500 [04:50<02:25,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1685/2500 [04:50<02:18,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1688/2500 [04:50<01:39,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1690/2500 [04:51<01:59,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1692/2500 [04:51<01:37,  8.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1694/2500 [04:51<02:03,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1697/2500 [04:52<02:16,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1700/2500 [04:52<01:38,  8.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1701/2500 [04:52<02:02,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1704/2500 [04:53<02:28,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1705/2500 [04:53<02:33,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1711/2500 [04:54<01:19,  9.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 1713/2500 [04:54<01:33,  8.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 1715/2500 [04:55<02:07,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 1718/2500 [04:56<02:44,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1719/2500 [04:56<03:33,  3.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1723/2500 [04:56<02:07,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1724/2500 [04:57<02:21,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 1730/2500 [04:57<01:20,  9.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1732/2500 [04:57<01:18,  9.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1734/2500 [04:58<01:42,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1736/2500 [04:58<01:54,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1737/2500 [04:59<02:47,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 1742/2500 [04:59<01:49,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 1744/2500 [04:59<01:34,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 1746/2500 [05:00<01:38,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 1748/2500 [05:00<01:37,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1750/2500 [05:00<01:31,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 1754/2500 [05:01<01:45,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 1756/2500 [05:01<02:07,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1757/2500 [05:02<03:01,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1758/2500 [05:02<03:41,  3.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 1760/2500 [05:03<03:10,  3.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1763/2500 [05:03<01:52,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1764/2500 [05:03<02:13,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1768/2500 [05:04<01:24,  8.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1770/2500 [05:04<02:10,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1771/2500 [05:05<02:33,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1774/2500 [05:05<01:49,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1776/2500 [05:05<01:44,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1777/2500 [05:05<01:56,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1779/2500 [05:06<02:08,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1781/2500 [05:06<01:53,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 1786/2500 [05:06<01:37,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1789/2500 [05:07<01:32,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1790/2500 [05:07<01:51,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1793/2500 [05:08<02:01,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1796/2500 [05:08<01:32,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1798/2500 [05:08<01:28,  7.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1800/2500 [05:09<01:42,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1802/2500 [05:09<01:32,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1803/2500 [05:09<02:03,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1807/2500 [05:10<01:50,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1809/2500 [05:10<01:43,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1811/2500 [05:11<02:05,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1813/2500 [05:11<02:12,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1815/2500 [05:11<01:46,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1816/2500 [05:12<02:13,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1818/2500 [05:12<02:02,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1821/2500 [05:12<01:40,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1824/2500 [05:13<01:30,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1827/2500 [05:13<01:18,  8.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1830/2500 [05:14<01:40,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1832/2500 [05:14<01:22,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1835/2500 [05:15<02:28,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▎  | 1840/2500 [05:15<01:13,  8.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 1842/2500 [05:16<02:22,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1848/2500 [05:17<01:34,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1850/2500 [05:17<01:44,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1854/2500 [05:17<01:27,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1855/2500 [05:17<01:26,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1858/2500 [05:18<01:44,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1860/2500 [05:18<01:37,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1862/2500 [05:19<02:12,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 1865/2500 [05:19<01:24,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 1868/2500 [05:20<01:36,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 1872/2500 [05:20<01:18,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1875/2500 [05:20<00:57, 10.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1877/2500 [05:21<01:28,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 1880/2500 [05:22<02:00,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 1884/2500 [05:22<01:57,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 1887/2500 [05:23<01:20,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1889/2500 [05:23<01:19,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1891/2500 [05:23<01:18,  7.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1896/2500 [05:23<00:49, 12.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1898/2500 [05:24<01:21,  7.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1900/2500 [05:25<02:16,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1901/2500 [05:25<02:20,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1903/2500 [05:26<02:09,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▋  | 1912/2500 [05:26<00:57, 10.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1914/2500 [05:27<01:02,  9.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1916/2500 [05:27<01:25,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1919/2500 [05:28<01:52,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1923/2500 [05:29<01:30,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1925/2500 [05:29<01:47,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1928/2500 [05:29<01:25,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1930/2500 [05:30<01:08,  8.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1932/2500 [05:30<01:27,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1934/2500 [05:30<01:28,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1937/2500 [05:31<01:39,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1939/2500 [05:31<01:31,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1940/2500 [05:32<01:45,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1942/2500 [05:32<02:03,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1946/2500 [05:33<01:29,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1947/2500 [05:33<01:31,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1949/2500 [05:33<01:36,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1953/2500 [05:33<01:05,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1955/2500 [05:34<01:07,  8.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1958/2500 [05:34<01:11,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1960/2500 [05:35<01:33,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 1963/2500 [05:35<01:49,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 1965/2500 [05:36<01:39,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 1968/2500 [05:36<01:27,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1971/2500 [05:37<01:13,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1974/2500 [05:37<01:06,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1975/2500 [05:37<01:11,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1978/2500 [05:37<01:07,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1980/2500 [05:38<01:35,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1981/2500 [05:38<01:41,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1983/2500 [05:39<02:05,  4.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1986/2500 [05:39<01:18,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 1990/2500 [05:39<00:48, 10.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 1992/2500 [05:40<01:14,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 1996/2500 [05:41<01:24,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 1998/2500 [05:41<01:16,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2001/2500 [05:42<01:31,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2007/2500 [05:42<00:53,  9.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2009/2500 [05:42<00:54,  9.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2011/2500 [05:43<01:45,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2012/2500 [05:44<01:49,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2015/2500 [05:44<01:27,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2018/2500 [05:44<01:05,  7.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2022/2500 [05:45<00:49,  9.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2024/2500 [05:45<01:07,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2027/2500 [05:45<00:57,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2029/2500 [05:46<01:08,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2031/2500 [05:46<01:26,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2032/2500 [05:47<01:56,  4.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2037/2500 [05:47<01:14,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2040/2500 [05:48<00:56,  8.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2042/2500 [05:48<00:55,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2043/2500 [05:48<01:34,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2045/2500 [05:49<01:24,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2049/2500 [05:49<00:56,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2052/2500 [05:50<01:03,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2053/2500 [05:50<01:54,  3.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2057/2500 [05:51<01:09,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2059/2500 [05:51<01:04,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2061/2500 [05:52<01:25,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2063/2500 [05:52<01:32,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2065/2500 [05:53<01:40,  4.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2067/2500 [05:53<01:21,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2070/2500 [05:54<01:33,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2072/2500 [05:54<01:14,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2076/2500 [05:54<00:53,  7.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2078/2500 [05:55<01:00,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2079/2500 [05:55<01:13,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2081/2500 [05:55<01:15,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2084/2500 [05:56<01:08,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2087/2500 [05:56<00:52,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2090/2500 [05:57<01:07,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2093/2500 [05:57<01:08,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2096/2500 [05:58<00:48,  8.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2098/2500 [05:58<01:08,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2101/2500 [05:59<01:08,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2103/2500 [05:59<01:10,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2107/2500 [06:00<01:14,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2111/2500 [06:00<00:46,  8.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2113/2500 [06:01<00:40,  9.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2115/2500 [06:01<00:46,  8.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2117/2500 [06:01<00:57,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2120/2500 [06:02<00:54,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2122/2500 [06:02<01:13,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2123/2500 [06:03<01:16,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2124/2500 [06:03<01:23,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2127/2500 [06:03<01:16,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2129/2500 [06:04<01:08,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2136/2500 [06:04<00:36,  9.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2138/2500 [06:05<00:48,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2140/2500 [06:05<00:52,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2144/2500 [06:06<01:00,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2146/2500 [06:06<01:04,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2147/2500 [06:07<01:21,  4.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2149/2500 [06:07<01:12,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2152/2500 [06:08<00:50,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2154/2500 [06:08<00:44,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2156/2500 [06:08<00:35,  9.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▋ | 2161/2500 [06:09<00:53,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2163/2500 [06:09<01:04,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2164/2500 [06:10<01:23,  4.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2165/2500 [06:10<01:21,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2166/2500 [06:11<01:30,  3.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2171/2500 [06:11<00:45,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2174/2500 [06:11<00:42,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2176/2500 [06:12<00:48,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2180/2500 [06:12<00:39,  8.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2182/2500 [06:12<00:45,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2184/2500 [06:13<00:54,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2185/2500 [06:13<01:03,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2188/2500 [06:14<00:59,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2189/2500 [06:14<00:55,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2190/2500 [06:14<01:09,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2192/2500 [06:15<01:08,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2193/2500 [06:15<01:07,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2196/2500 [06:15<00:51,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2198/2500 [06:16<01:01,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2199/2500 [06:16<01:14,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2200/2500 [06:17<01:29,  3.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2203/2500 [06:17<00:56,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2206/2500 [06:17<00:43,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2208/2500 [06:18<00:47,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2209/2500 [06:18<00:53,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2210/2500 [06:19<01:21,  3.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2211/2500 [06:19<01:22,  3.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2212/2500 [06:19<01:24,  3.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2214/2500 [06:20<01:05,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 2216/2500 [06:20<01:02,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2217/2500 [06:20<00:54,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2219/2500 [06:20<00:53,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2222/2500 [06:21<00:57,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2224/2500 [06:21<00:48,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2226/2500 [06:22<00:50,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2228/2500 [06:22<00:38,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2229/2500 [06:22<00:43,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2231/2500 [06:23<00:49,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2232/2500 [06:23<01:06,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2234/2500 [06:23<00:57,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2237/2500 [06:24<00:46,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2242/2500 [06:24<00:26,  9.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2244/2500 [06:25<00:34,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2249/2500 [06:25<00:32,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2250/2500 [06:26<00:51,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2251/2500 [06:26<00:57,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2255/2500 [06:27<00:41,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2257/2500 [06:27<00:39,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2258/2500 [06:27<00:36,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2262/2500 [06:28<00:26,  8.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2266/2500 [06:28<00:23,  9.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2268/2500 [06:28<00:26,  8.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2270/2500 [06:29<00:24,  9.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2272/2500 [06:30<00:59,  3.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2273/2500 [06:30<01:00,  3.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2276/2500 [06:31<00:56,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2279/2500 [06:31<00:38,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2282/2500 [06:31<00:33,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 2285/2500 [06:32<00:24,  8.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 2287/2500 [06:32<00:20, 10.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2289/2500 [06:32<00:32,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2290/2500 [06:32<00:33,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2291/2500 [06:33<00:36,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2293/2500 [06:33<00:42,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2294/2500 [06:33<00:36,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2297/2500 [06:34<00:34,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2299/2500 [06:34<00:25,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2302/2500 [06:35<00:33,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2303/2500 [06:35<00:36,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2305/2500 [06:36<00:48,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2310/2500 [06:36<00:26,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2312/2500 [06:37<00:37,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2313/2500 [06:37<00:38,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2317/2500 [06:37<00:25,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2318/2500 [06:37<00:24,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2322/2500 [06:38<00:26,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2323/2500 [06:38<00:26,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2324/2500 [06:39<00:52,  3.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2325/2500 [06:39<00:55,  3.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2326/2500 [06:40<00:53,  3.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2331/2500 [06:40<00:24,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2332/2500 [06:40<00:27,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2335/2500 [06:41<00:24,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2337/2500 [06:41<00:21,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▎| 2339/2500 [06:41<00:22,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2341/2500 [06:41<00:20,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2342/2500 [06:42<00:25,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2344/2500 [06:43<00:39,  3.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2346/2500 [06:43<00:32,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2349/2500 [06:43<00:27,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2350/2500 [06:44<00:36,  4.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2353/2500 [06:44<00:26,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2355/2500 [06:45<00:33,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2359/2500 [06:45<00:25,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2362/2500 [06:46<00:23,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2367/2500 [06:47<00:19,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2369/2500 [06:47<00:19,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2371/2500 [06:47<00:15,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2372/2500 [06:47<00:22,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2373/2500 [06:48<00:26,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2375/2500 [06:48<00:24,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2376/2500 [06:48<00:21,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2377/2500 [06:49<00:35,  3.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2379/2500 [06:49<00:26,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2381/2500 [06:49<00:21,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2382/2500 [06:50<00:28,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2385/2500 [06:50<00:18,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2387/2500 [06:50<00:17,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2389/2500 [06:51<00:20,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2391/2500 [06:51<00:22,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2395/2500 [06:52<00:15,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2397/2500 [06:52<00:15,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2400/2500 [06:53<00:16,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2402/2500 [06:53<00:12,  7.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2406/2500 [06:53<00:12,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2407/2500 [06:54<00:19,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2408/2500 [06:54<00:20,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2410/2500 [06:55<00:22,  4.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2412/2500 [06:55<00:19,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2416/2500 [06:55<00:12,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2418/2500 [06:56<00:09,  8.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2420/2500 [06:56<00:09,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2422/2500 [06:56<00:08,  8.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2427/2500 [06:57<00:09,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2429/2500 [06:58<00:17,  4.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2430/2500 [06:58<00:17,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2434/2500 [06:59<00:12,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2437/2500 [06:59<00:09,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2439/2500 [06:59<00:10,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2442/2500 [07:00<00:09,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2445/2500 [07:00<00:06,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2447/2500 [07:01<00:07,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2449/2500 [07:01<00:07,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2451/2500 [07:01<00:09,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2455/2500 [07:02<00:06,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2456/2500 [07:02<00:07,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2457/2500 [07:03<00:08,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2458/2500 [07:03<00:08,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2459/2500 [07:03<00:08,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2461/2500 [07:03<00:09,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▊| 2464/2500 [07:04<00:05,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▊| 2467/2500 [07:04<00:04,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2469/2500 [07:04<00:03,  9.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2473/2500 [07:05<00:03,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2475/2500 [07:06<00:06,  4.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2479/2500 [07:06<00:03,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2481/2500 [07:06<00:02,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2483/2500 [07:07<00:02,  7.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2485/2500 [07:07<00:02,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2487/2500 [07:07<00:01,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2490/2500 [07:08<00:01,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2492/2500 [07:08<00:01,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2494/2500 [07:09<00:01,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2495/2500 [07:09<00:00,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2498/2500 [07:09<00:00,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 2500/2500 [07:10<00:00,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 1/2500 [00:02<1:48:23,  2.60s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 5/2500 [00:02<15:11,  2.74it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 10/2500 [00:03<06:32,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 16/2500 [00:03<03:25, 12.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 19/2500 [00:03<03:20, 12.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 21/2500 [00:05<11:21,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 23/2500 [00:05<09:31,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 26/2500 [00:06<09:35,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 29/2500 [00:06<06:25,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 31/2500 [00:06<06:14,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 34/2500 [00:06<04:36,  8.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 36/2500 [00:07<04:35,  8.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 38/2500 [00:07<05:44,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 41/2500 [00:08<07:18,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 42/2500 [00:08<09:34,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 45/2500 [00:09<07:10,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 46/2500 [00:09<07:50,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 47/2500 [00:09<08:04,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 51/2500 [00:10<06:18,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 53/2500 [00:10<05:14,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 58/2500 [00:10<04:36,  8.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 60/2500 [00:11<06:13,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 62/2500 [00:12<09:12,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 66/2500 [00:12<06:29,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 69/2500 [00:12<06:15,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 70/2500 [00:13<06:32,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 71/2500 [00:13<06:55,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 74/2500 [00:13<05:16,  7.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 75/2500 [00:14<07:38,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 76/2500 [00:14<08:18,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 80/2500 [00:14<05:22,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 82/2500 [00:15<06:42,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 84/2500 [00:15<06:46,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 85/2500 [00:15<07:27,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 89/2500 [00:16<06:02,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 92/2500 [00:16<04:35,  8.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 94/2500 [00:17<07:52,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 96/2500 [00:17<06:58,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 99/2500 [00:18<07:04,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 101/2500 [00:18<06:46,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 103/2500 [00:18<06:38,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 104/2500 [00:18<06:44,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 106/2500 [00:19<07:28,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 109/2500 [00:19<06:08,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 112/2500 [00:20<05:52,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 114/2500 [00:20<04:54,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 117/2500 [00:20<05:00,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 118/2500 [00:21<07:12,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 121/2500 [00:21<06:38,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 123/2500 [00:21<06:02,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 125/2500 [00:22<06:46,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 128/2500 [00:22<06:28,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 129/2500 [00:22<05:56,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 131/2500 [00:23<05:15,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 132/2500 [00:23<05:59,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 134/2500 [00:23<05:19,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 136/2500 [00:24<08:18,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 139/2500 [00:24<05:30,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 141/2500 [00:24<04:57,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 145/2500 [00:25<06:16,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 147/2500 [00:25<05:45,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 149/2500 [00:26<06:52,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 152/2500 [00:26<05:40,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 153/2500 [00:26<06:37,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 157/2500 [00:27<04:50,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 159/2500 [00:27<05:14,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 160/2500 [00:27<05:15,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 161/2500 [00:28<07:24,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 165/2500 [00:28<04:44,  8.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 166/2500 [00:28<04:59,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 168/2500 [00:28<06:06,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 169/2500 [00:29<07:01,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 173/2500 [00:30<07:18,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 176/2500 [00:30<05:12,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 178/2500 [00:30<05:28,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 181/2500 [00:31<05:19,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 184/2500 [00:31<04:28,  8.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 186/2500 [00:31<05:43,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 188/2500 [00:32<06:39,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 189/2500 [00:32<10:10,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 190/2500 [00:33<10:48,  3.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 196/2500 [00:33<06:31,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 199/2500 [00:34<05:17,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 201/2500 [00:34<05:13,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 203/2500 [00:34<06:26,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 207/2500 [00:35<04:44,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 208/2500 [00:35<05:28,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 212/2500 [00:36<05:31,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 214/2500 [00:36<04:43,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 216/2500 [00:37<06:49,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▊         | 218/2500 [00:37<06:31,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 220/2500 [00:37<06:47,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 227/2500 [00:38<04:26,  8.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 228/2500 [00:38<04:44,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 230/2500 [00:39<06:18,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 232/2500 [00:39<07:52,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 233/2500 [00:39<07:09,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 235/2500 [00:40<08:20,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 237/2500 [00:40<06:49,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 239/2500 [00:40<05:58,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 242/2500 [00:41<04:17,  8.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 244/2500 [00:41<04:48,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 246/2500 [00:41<04:50,  7.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 248/2500 [00:42<07:29,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 249/2500 [00:42<06:35,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 253/2500 [00:43<08:26,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 256/2500 [00:44<07:00,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 258/2500 [00:44<05:28,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 260/2500 [00:44<05:57,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 262/2500 [00:44<06:42,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 266/2500 [00:45<04:55,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 269/2500 [00:45<05:01,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 270/2500 [00:46<07:20,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 272/2500 [00:46<06:48,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 276/2500 [00:47<06:12,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 279/2500 [00:47<04:53,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 281/2500 [00:47<05:54,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 282/2500 [00:48<06:40,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 283/2500 [00:48<07:28,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 285/2500 [00:48<06:38,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 289/2500 [00:49<04:34,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 291/2500 [00:49<05:12,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 293/2500 [00:49<04:35,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 297/2500 [00:50<05:02,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 300/2500 [00:51<07:12,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 304/2500 [00:52<06:53,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 306/2500 [00:52<06:01,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 308/2500 [00:52<05:47,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 310/2500 [00:53<06:38,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 312/2500 [00:53<06:30,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 316/2500 [00:53<04:17,  8.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 317/2500 [00:54<06:52,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 319/2500 [00:54<06:51,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 321/2500 [00:55<07:38,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 324/2500 [00:55<05:42,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 325/2500 [00:55<07:50,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 327/2500 [00:56<08:32,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 330/2500 [00:56<05:05,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 333/2500 [00:57<05:19,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 336/2500 [00:57<05:51,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 339/2500 [00:58<05:32,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 342/2500 [00:58<05:28,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 343/2500 [00:58<05:47,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 347/2500 [00:59<05:29,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 351/2500 [00:59<03:57,  9.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 353/2500 [01:00<05:11,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 355/2500 [01:00<04:34,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 357/2500 [01:01<06:41,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 360/2500 [01:01<05:16,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 362/2500 [01:01<05:01,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 364/2500 [01:02<06:38,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 367/2500 [01:02<04:59,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 369/2500 [01:02<05:46,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 370/2500 [01:03<07:38,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 374/2500 [01:03<05:41,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 376/2500 [01:04<06:06,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 377/2500 [01:04<05:51,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 381/2500 [01:04<04:36,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 386/2500 [01:05<03:11, 11.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 389/2500 [01:06<07:03,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 393/2500 [01:06<04:48,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 396/2500 [01:07<07:19,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 398/2500 [01:08<06:27,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 402/2500 [01:08<05:23,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 407/2500 [01:08<03:20, 10.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 411/2500 [01:09<05:46,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 413/2500 [01:10<05:35,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 417/2500 [01:10<04:58,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 424/2500 [01:11<04:32,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 426/2500 [01:12<04:29,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 428/2500 [01:12<04:54,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 430/2500 [01:13<06:52,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 433/2500 [01:13<05:23,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 435/2500 [01:13<05:14,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 437/2500 [01:13<04:58,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 438/2500 [01:14<05:47,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 439/2500 [01:14<06:40,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 440/2500 [01:14<06:51,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 445/2500 [01:15<03:53,  8.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 447/2500 [01:15<04:53,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 448/2500 [01:15<05:15,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 449/2500 [01:16<09:10,  3.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 450/2500 [01:16<09:29,  3.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 451/2500 [01:17<09:57,  3.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 456/2500 [01:17<05:49,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 458/2500 [01:18<06:32,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 459/2500 [01:18<06:59,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 460/2500 [01:18<07:04,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 462/2500 [01:18<05:53,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 463/2500 [01:19<06:12,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 468/2500 [01:19<04:21,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 470/2500 [01:20<05:26,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 471/2500 [01:20<07:07,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 475/2500 [01:21<05:38,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 476/2500 [01:21<06:04,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 477/2500 [01:21<06:31,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 481/2500 [01:21<03:59,  8.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 483/2500 [01:22<03:36,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 486/2500 [01:22<05:46,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 488/2500 [01:23<05:54,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 491/2500 [01:23<05:45,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 493/2500 [01:24<06:10,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 494/2500 [01:24<05:46,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 497/2500 [01:24<04:27,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 500/2500 [01:25<03:41,  9.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 501/2500 [01:25<05:17,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 503/2500 [01:26<07:35,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 505/2500 [01:26<08:17,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 507/2500 [01:26<07:25,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 511/2500 [01:27<04:44,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 514/2500 [01:27<05:02,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 517/2500 [01:28<04:28,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 518/2500 [01:28<06:48,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 522/2500 [01:29<05:03,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 523/2500 [01:29<04:58,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 526/2500 [01:30<06:28,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 528/2500 [01:30<05:16,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 530/2500 [01:30<05:39,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 535/2500 [01:31<03:50,  8.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 537/2500 [01:31<03:19,  9.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 541/2500 [01:32<04:52,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 543/2500 [01:32<06:07,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 544/2500 [01:33<06:26,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 546/2500 [01:33<05:56,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 547/2500 [01:33<07:11,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 548/2500 [01:34<08:57,  3.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 551/2500 [01:34<06:31,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 555/2500 [01:35<05:49,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 559/2500 [01:35<03:55,  8.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 562/2500 [01:36<05:12,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 564/2500 [01:36<05:35,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 565/2500 [01:37<06:46,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 566/2500 [01:37<07:29,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 568/2500 [01:37<08:21,  3.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 571/2500 [01:38<06:31,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 574/2500 [01:38<04:13,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 577/2500 [01:38<03:00, 10.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 579/2500 [01:39<04:28,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 583/2500 [01:39<03:50,  8.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 585/2500 [01:40<06:35,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 587/2500 [01:41<07:22,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 588/2500 [01:41<07:27,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 590/2500 [01:41<05:54,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 591/2500 [01:42<07:28,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 594/2500 [01:42<06:27,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 595/2500 [01:42<06:27,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 600/2500 [01:43<04:32,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 601/2500 [01:43<04:26,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 602/2500 [01:43<05:11,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 606/2500 [01:44<04:29,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 609/2500 [01:45<06:03,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 610/2500 [01:45<05:31,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 611/2500 [01:45<06:19,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 613/2500 [01:46<06:48,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 615/2500 [01:46<04:35,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 617/2500 [01:46<05:17,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 618/2500 [01:46<04:56,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 621/2500 [01:47<04:27,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 623/2500 [01:47<05:53,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 625/2500 [01:48<06:53,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 627/2500 [01:48<05:54,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 629/2500 [01:49<05:40,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 635/2500 [01:49<02:48, 11.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 637/2500 [01:49<04:04,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 639/2500 [01:50<06:04,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 640/2500 [01:51<07:59,  3.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 642/2500 [01:51<07:46,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 644/2500 [01:51<06:03,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 646/2500 [01:52<04:42,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 651/2500 [01:52<03:20,  9.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 653/2500 [01:52<03:26,  8.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 655/2500 [01:52<03:37,  8.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 656/2500 [01:53<04:59,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 658/2500 [01:53<05:41,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 659/2500 [01:54<08:34,  3.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 661/2500 [01:54<07:19,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 664/2500 [01:55<05:42,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 668/2500 [01:55<03:06,  9.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 670/2500 [01:55<02:59, 10.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 672/2500 [01:55<03:10,  9.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 674/2500 [01:56<03:59,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 676/2500 [01:57<06:24,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 677/2500 [01:57<06:32,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 679/2500 [01:57<05:28,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 680/2500 [01:57<06:23,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 684/2500 [01:58<04:30,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 687/2500 [01:58<03:08,  9.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 689/2500 [01:58<03:20,  9.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 691/2500 [01:59<05:19,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 693/2500 [01:59<04:27,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 697/2500 [02:00<04:09,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 699/2500 [02:00<05:09,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 701/2500 [02:01<05:10,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 702/2500 [02:01<04:56,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 704/2500 [02:01<05:08,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 707/2500 [02:02<05:10,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 710/2500 [02:02<03:49,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 715/2500 [02:02<03:04,  9.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 717/2500 [02:03<06:15,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 720/2500 [02:04<05:04,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 722/2500 [02:04<05:14,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 724/2500 [02:04<04:29,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 726/2500 [02:05<06:15,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 728/2500 [02:05<05:03,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 731/2500 [02:05<03:42,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 732/2500 [02:06<04:17,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 734/2500 [02:06<04:43,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 737/2500 [02:06<04:57,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 742/2500 [02:07<03:10,  9.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 744/2500 [02:08<05:15,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 746/2500 [02:08<06:28,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 747/2500 [02:09<07:13,  4.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 755/2500 [02:10<03:58,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 759/2500 [02:10<03:17,  8.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 761/2500 [02:10<04:23,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 763/2500 [02:11<05:45,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 765/2500 [02:11<05:42,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 767/2500 [02:12<06:13,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 769/2500 [02:12<05:18,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 772/2500 [02:12<04:08,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 773/2500 [02:13<04:28,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 776/2500 [02:13<04:03,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 777/2500 [02:14<05:30,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 779/2500 [02:14<05:05,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 784/2500 [02:14<03:46,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 787/2500 [02:15<04:30,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 790/2500 [02:15<04:13,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 791/2500 [02:16<04:22,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 793/2500 [02:16<04:00,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 795/2500 [02:17<06:20,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 797/2500 [02:17<05:16,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 799/2500 [02:17<04:42,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 801/2500 [02:18<06:07,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 804/2500 [02:18<05:05,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 807/2500 [02:19<04:12,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 809/2500 [02:19<04:28,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 812/2500 [02:19<03:31,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 813/2500 [02:19<04:04,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 816/2500 [02:20<04:00,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 819/2500 [02:20<03:23,  8.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 820/2500 [02:20<03:45,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 822/2500 [02:21<03:59,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 824/2500 [02:21<04:43,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 826/2500 [02:22<05:38,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 827/2500 [02:22<05:25,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 828/2500 [02:22<05:31,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 833/2500 [02:22<02:55,  9.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 838/2500 [02:23<02:43, 10.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 840/2500 [02:23<03:25,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 842/2500 [02:24<03:19,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 844/2500 [02:24<04:37,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 845/2500 [02:24<04:26,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 846/2500 [02:25<06:19,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 847/2500 [02:25<08:07,  3.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 850/2500 [02:26<05:26,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 854/2500 [02:26<03:16,  8.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 856/2500 [02:26<03:38,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 858/2500 [02:26<03:13,  8.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 860/2500 [02:27<03:38,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 862/2500 [02:27<05:07,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 863/2500 [02:28<04:44,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 864/2500 [02:28<05:47,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 865/2500 [02:28<06:12,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 868/2500 [02:29<04:17,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 873/2500 [02:29<02:29, 10.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 875/2500 [02:29<02:33, 10.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 877/2500 [02:29<02:44,  9.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 879/2500 [02:30<06:48,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 881/2500 [02:31<06:19,  4.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 882/2500 [02:31<07:11,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 889/2500 [02:32<03:18,  8.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 893/2500 [02:32<03:00,  8.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 895/2500 [02:33<03:53,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 897/2500 [02:33<03:33,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 899/2500 [02:33<05:04,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 901/2500 [02:34<06:31,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 902/2500 [02:34<06:15,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 903/2500 [02:35<06:31,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 910/2500 [02:35<02:51,  9.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 912/2500 [02:36<03:21,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 914/2500 [02:36<04:59,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 917/2500 [02:37<03:52,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 921/2500 [02:37<04:01,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 923/2500 [02:38<04:02,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 926/2500 [02:38<05:21,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 928/2500 [02:39<04:52,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 930/2500 [02:39<04:37,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 933/2500 [02:40<05:05,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 935/2500 [02:40<04:19,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 937/2500 [02:40<03:50,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 939/2500 [02:40<03:02,  8.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 941/2500 [02:41<04:54,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 943/2500 [02:41<04:54,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 947/2500 [02:42<04:01,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 948/2500 [02:42<04:39,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 952/2500 [02:43<03:45,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 954/2500 [02:43<03:14,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 956/2500 [02:43<03:30,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 957/2500 [02:44<04:47,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 958/2500 [02:44<05:32,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 961/2500 [02:44<05:00,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 964/2500 [02:45<04:29,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 965/2500 [02:45<04:53,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 968/2500 [02:46<04:31,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 969/2500 [02:46<04:29,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 971/2500 [02:46<03:43,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 974/2500 [02:47<04:41,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 975/2500 [02:47<05:24,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 978/2500 [02:48<05:09,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 979/2500 [02:48<05:52,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 981/2500 [02:48<04:48,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 983/2500 [02:49<04:46,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 986/2500 [02:49<03:07,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 989/2500 [02:49<02:52,  8.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 990/2500 [02:49<03:23,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 991/2500 [02:50<06:21,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 993/2500 [02:50<05:34,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 994/2500 [02:51<05:32,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 995/2500 [02:51<05:36,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 998/2500 [02:51<05:05,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1000/2500 [02:52<04:33,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1002/2500 [02:52<03:58,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1005/2500 [02:52<03:21,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1006/2500 [02:53<03:57,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1010/2500 [02:53<04:02,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1012/2500 [02:54<04:37,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1013/2500 [02:54<05:48,  4.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1015/2500 [02:55<05:03,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1016/2500 [02:55<05:06,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1019/2500 [02:55<03:57,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1021/2500 [02:55<04:16,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1023/2500 [02:56<04:28,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1026/2500 [02:56<03:58,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1027/2500 [02:57<04:15,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1028/2500 [02:57<04:45,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1030/2500 [02:57<04:42,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1031/2500 [02:57<04:48,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████▏     | 1034/2500 [02:58<05:45,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1036/2500 [02:59<05:33,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1037/2500 [02:59<08:04,  3.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1042/2500 [03:00<04:14,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1044/2500 [03:00<03:28,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1046/2500 [03:00<03:32,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1050/2500 [03:01<02:55,  8.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1052/2500 [03:02<07:23,  3.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1054/2500 [03:03<06:42,  3.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1057/2500 [03:03<05:10,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1058/2500 [03:03<05:03,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1062/2500 [03:04<03:32,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1065/2500 [03:04<02:52,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1066/2500 [03:04<03:24,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1067/2500 [03:04<04:15,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1068/2500 [03:05<04:24,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1070/2500 [03:05<04:22,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1072/2500 [03:05<02:58,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1074/2500 [03:06<07:28,  3.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1078/2500 [03:07<04:51,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1081/2500 [03:07<03:28,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1083/2500 [03:07<02:59,  7.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1085/2500 [03:08<03:35,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1086/2500 [03:08<04:50,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1088/2500 [03:08<04:08,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1090/2500 [03:09<03:37,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▎     | 1092/2500 [03:09<03:55,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1093/2500 [03:09<03:42,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1094/2500 [03:10<04:53,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1096/2500 [03:10<04:40,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1098/2500 [03:10<04:35,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1101/2500 [03:11<04:23,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1104/2500 [03:11<03:49,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1105/2500 [03:11<04:06,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1108/2500 [03:12<03:28,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1109/2500 [03:12<04:42,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1111/2500 [03:13<04:11,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1112/2500 [03:13<05:28,  4.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1114/2500 [03:13<04:52,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1117/2500 [03:14<03:45,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1121/2500 [03:14<02:47,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1123/2500 [03:15<04:41,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1126/2500 [03:15<03:26,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1128/2500 [03:15<03:41,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1130/2500 [03:16<03:32,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1132/2500 [03:16<03:39,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1135/2500 [03:17<04:24,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1139/2500 [03:17<02:48,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1141/2500 [03:17<02:38,  8.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1143/2500 [03:18<04:00,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1146/2500 [03:18<03:36,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1149/2500 [03:19<04:04,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1150/2500 [03:19<04:13,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1151/2500 [03:20<05:32,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1154/2500 [03:20<04:12,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1156/2500 [03:20<04:08,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1159/2500 [03:21<03:06,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1161/2500 [03:21<03:07,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1163/2500 [03:22<04:07,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1165/2500 [03:22<03:48,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1166/2500 [03:22<03:35,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1169/2500 [03:22<03:34,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1171/2500 [03:23<03:52,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1175/2500 [03:23<03:03,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1176/2500 [03:24<03:44,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1179/2500 [03:24<03:23,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1183/2500 [03:25<02:48,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1184/2500 [03:25<02:58,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1185/2500 [03:25<05:14,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1186/2500 [03:26<05:45,  3.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1188/2500 [03:26<05:29,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1193/2500 [03:26<02:40,  8.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1196/2500 [03:27<01:58, 11.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1200/2500 [03:27<02:11,  9.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1202/2500 [03:27<02:24,  8.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1204/2500 [03:28<04:15,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1205/2500 [03:29<05:59,  3.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1207/2500 [03:29<04:50,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1210/2500 [03:30<04:14,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1212/2500 [03:30<03:22,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1215/2500 [03:30<02:26,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1217/2500 [03:30<02:25,  8.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1220/2500 [03:31<04:04,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1224/2500 [03:32<03:04,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1226/2500 [03:32<03:30,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1228/2500 [03:32<03:08,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1229/2500 [03:32<03:09,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1232/2500 [03:33<03:08,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1233/2500 [03:33<02:59,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1236/2500 [03:33<02:43,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1237/2500 [03:33<03:10,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1238/2500 [03:34<03:56,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1241/2500 [03:34<03:53,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1242/2500 [03:35<03:36,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1243/2500 [03:35<04:26,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1250/2500 [03:35<02:16,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1252/2500 [03:36<02:02, 10.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1254/2500 [03:36<02:25,  8.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1257/2500 [03:37<03:47,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1261/2500 [03:37<03:22,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1262/2500 [03:38<03:09,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1265/2500 [03:38<04:08,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1270/2500 [03:39<03:00,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1272/2500 [03:39<02:52,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1276/2500 [03:40<02:46,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1278/2500 [03:40<02:49,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1280/2500 [03:40<02:26,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 1282/2500 [03:41<02:40,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 1283/2500 [03:41<03:21,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 1284/2500 [03:41<04:30,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1287/2500 [03:42<03:42,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1288/2500 [03:42<03:22,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1292/2500 [03:42<02:50,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1294/2500 [03:43<05:04,  3.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1297/2500 [03:44<03:50,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1302/2500 [03:44<02:05,  9.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1304/2500 [03:44<02:05,  9.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1306/2500 [03:45<02:43,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1308/2500 [03:45<02:46,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1310/2500 [03:45<02:36,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1313/2500 [03:46<03:08,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1314/2500 [03:46<03:44,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1315/2500 [03:47<05:45,  3.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1317/2500 [03:47<04:18,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1320/2500 [03:48<03:31,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1323/2500 [03:48<02:20,  8.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1325/2500 [03:48<02:15,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1327/2500 [03:49<03:36,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1329/2500 [03:49<04:38,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1330/2500 [03:50<05:21,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1334/2500 [03:50<03:11,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1337/2500 [03:51<02:53,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1340/2500 [03:51<01:58,  9.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1342/2500 [03:51<01:50, 10.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1344/2500 [03:51<02:35,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1346/2500 [03:52<03:26,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1350/2500 [03:53<03:31,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1352/2500 [03:53<03:13,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1354/2500 [03:53<02:35,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1358/2500 [03:53<02:03,  9.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1361/2500 [03:55<03:43,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1365/2500 [03:55<02:57,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1366/2500 [03:55<03:20,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1369/2500 [03:56<03:40,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1373/2500 [03:56<02:56,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1376/2500 [03:57<02:24,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1378/2500 [03:57<02:42,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1380/2500 [03:58<04:50,  3.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1381/2500 [03:58<05:07,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1385/2500 [03:59<02:45,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1387/2500 [03:59<02:24,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1389/2500 [03:59<02:42,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1393/2500 [04:00<02:22,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1397/2500 [04:00<02:26,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1399/2500 [04:01<03:25,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1401/2500 [04:02<03:48,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1403/2500 [04:02<03:37,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1406/2500 [04:02<02:58,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1407/2500 [04:02<02:44,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1410/2500 [04:03<02:37,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1411/2500 [04:03<02:58,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1412/2500 [04:03<03:14,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1414/2500 [04:03<02:44,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1416/2500 [04:04<02:36,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1418/2500 [04:04<02:40,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1419/2500 [04:05<04:37,  3.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1420/2500 [04:05<05:04,  3.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1423/2500 [04:06<04:11,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1426/2500 [04:06<02:58,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1429/2500 [04:07<02:49,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1431/2500 [04:07<02:42,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1432/2500 [04:07<02:57,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1434/2500 [04:08<03:28,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1435/2500 [04:08<06:07,  2.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1437/2500 [04:09<04:36,  3.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1439/2500 [04:09<03:53,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1441/2500 [04:10<03:52,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1444/2500 [04:10<02:59,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1446/2500 [04:10<02:33,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1447/2500 [04:11<06:10,  2.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1449/2500 [04:12<05:41,  3.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1450/2500 [04:12<04:56,  3.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1453/2500 [04:12<03:43,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1457/2500 [04:13<02:45,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1459/2500 [04:14<04:14,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1461/2500 [04:14<04:57,  3.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1462/2500 [04:15<05:02,  3.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▊    | 1464/2500 [04:15<04:09,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1465/2500 [04:15<04:05,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1468/2500 [04:16<03:08,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1469/2500 [04:16<03:11,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1470/2500 [04:16<03:28,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1471/2500 [04:17<07:37,  2.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1474/2500 [04:18<04:27,  3.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1475/2500 [04:18<03:50,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1477/2500 [04:18<03:06,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1478/2500 [04:18<03:33,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1480/2500 [04:19<03:18,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1481/2500 [04:19<03:46,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1482/2500 [04:21<09:09,  1.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1484/2500 [04:21<06:24,  2.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1486/2500 [04:22<06:24,  2.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1489/2500 [04:23<06:35,  2.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1490/2500 [04:23<06:16,  2.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1491/2500 [04:24<05:57,  2.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1493/2500 [04:24<04:29,  3.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1494/2500 [04:25<05:33,  3.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1495/2500 [04:25<07:12,  2.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1497/2500 [04:27<08:59,  1.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1498/2500 [04:27<07:50,  2.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1499/2500 [04:28<08:19,  2.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1502/2500 [04:29<09:06,  1.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1503/2500 [04:30<08:05,  2.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1504/2500 [04:30<07:07,  2.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1505/2500 [04:31<08:08,  2.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1507/2500 [04:33<12:32,  1.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1509/2500 [04:33<08:38,  1.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1511/2500 [04:34<06:52,  2.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1513/2500 [04:34<05:35,  2.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1516/2500 [04:37<08:04,  2.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1520/2500 [04:37<04:25,  3.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1521/2500 [04:38<07:20,  2.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1522/2500 [04:39<07:32,  2.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1525/2500 [04:40<06:37,  2.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1526/2500 [04:40<06:02,  2.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1527/2500 [04:41<08:21,  1.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1528/2500 [04:42<08:59,  1.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1529/2500 [04:42<07:26,  2.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1531/2500 [04:43<05:51,  2.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 1533/2500 [04:43<05:30,  2.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1534/2500 [04:43<04:51,  3.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 1536/2500 [04:44<05:49,  2.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1537/2500 [04:45<07:08,  2.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1538/2500 [04:46<07:03,  2.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1540/2500 [04:46<05:08,  3.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1541/2500 [04:46<05:25,  2.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1542/2500 [04:47<05:17,  3.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1545/2500 [04:47<03:55,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1546/2500 [04:48<04:07,  3.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1549/2500 [04:48<03:26,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1550/2500 [04:48<03:10,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1551/2500 [04:49<04:04,  3.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1553/2500 [04:49<03:22,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1554/2500 [04:49<03:46,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1557/2500 [04:50<04:07,  3.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1558/2500 [04:51<06:00,  2.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1559/2500 [04:51<05:23,  2.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1562/2500 [04:52<03:16,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1566/2500 [04:52<02:38,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1567/2500 [04:53<03:21,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1569/2500 [04:53<02:52,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1572/2500 [04:53<02:29,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1576/2500 [04:55<03:18,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1577/2500 [04:55<03:04,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1578/2500 [04:55<04:22,  3.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1579/2500 [04:56<04:30,  3.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1583/2500 [04:56<02:41,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1585/2500 [04:56<02:06,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1587/2500 [04:57<02:10,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1588/2500 [04:57<02:26,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1589/2500 [04:57<02:33,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1592/2500 [04:57<02:12,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1593/2500 [04:58<02:31,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1594/2500 [04:58<02:38,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1595/2500 [04:58<03:17,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1598/2500 [04:59<02:54,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1601/2500 [04:59<02:20,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1603/2500 [04:59<01:59,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1605/2500 [05:00<01:39,  9.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1608/2500 [05:01<04:02,  3.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1611/2500 [05:01<02:39,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1612/2500 [05:02<02:58,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1615/2500 [05:02<02:40,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1617/2500 [05:03<02:56,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1620/2500 [05:03<02:25,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1622/2500 [05:03<02:23,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1625/2500 [05:04<01:44,  8.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1626/2500 [05:04<02:52,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1628/2500 [05:04<02:19,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1631/2500 [05:05<02:59,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1634/2500 [05:05<02:27,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1635/2500 [05:06<02:53,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1636/2500 [05:06<02:53,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1642/2500 [05:06<01:27,  9.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1644/2500 [05:07<01:56,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1646/2500 [05:07<02:31,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1647/2500 [05:07<02:40,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1648/2500 [05:08<03:55,  3.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1652/2500 [05:08<02:10,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1653/2500 [05:09<02:06,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1656/2500 [05:09<02:01,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1659/2500 [05:09<01:55,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1661/2500 [05:10<01:47,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1662/2500 [05:10<01:57,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1664/2500 [05:10<02:24,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1666/2500 [05:11<02:27,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1667/2500 [05:11<02:40,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1670/2500 [05:12<02:28,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1674/2500 [05:12<01:55,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1675/2500 [05:12<02:36,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1679/2500 [05:13<01:55,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1681/2500 [05:13<01:53,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1683/2500 [05:14<02:05,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1686/2500 [05:14<02:09,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1688/2500 [05:15<02:39,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1691/2500 [05:15<02:13,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1692/2500 [05:15<02:52,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1693/2500 [05:16<03:27,  3.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1694/2500 [05:16<03:34,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1697/2500 [05:17<02:36,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1699/2500 [05:17<01:52,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1704/2500 [05:17<01:16, 10.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1706/2500 [05:18<02:58,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1710/2500 [05:19<02:14,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1712/2500 [05:19<01:47,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 1714/2500 [05:19<02:01,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 1716/2500 [05:19<02:03,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 1719/2500 [05:20<02:00,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1721/2500 [05:21<02:39,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 1724/2500 [05:21<02:05,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1725/2500 [05:21<02:18,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1728/2500 [05:21<02:06,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1729/2500 [05:22<02:13,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 1732/2500 [05:22<02:08,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 1734/2500 [05:23<02:10,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 1737/2500 [05:23<01:54,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 1739/2500 [05:23<01:38,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 1740/2500 [05:24<02:16,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 1742/2500 [05:24<02:04,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 1743/2500 [05:24<02:54,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 1745/2500 [05:25<02:22,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 1746/2500 [05:25<03:06,  4.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 1749/2500 [05:25<02:26,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1750/2500 [05:26<02:13,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 1752/2500 [05:26<02:18,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 1756/2500 [05:26<01:33,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1757/2500 [05:27<01:58,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1758/2500 [05:27<02:13,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1760/2500 [05:27<02:40,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1761/2500 [05:28<02:53,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1764/2500 [05:28<02:03,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1766/2500 [05:28<01:30,  8.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1770/2500 [05:29<01:39,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1774/2500 [05:29<01:10, 10.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1776/2500 [05:29<01:17,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1778/2500 [05:30<02:25,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1781/2500 [05:31<02:03,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 1783/2500 [05:31<01:38,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 1785/2500 [05:31<01:52,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1790/2500 [05:32<01:24,  8.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1792/2500 [05:32<01:21,  8.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1795/2500 [05:32<01:28,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1797/2500 [05:33<01:34,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1799/2500 [05:33<02:41,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1801/2500 [05:34<02:20,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1803/2500 [05:34<02:13,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1805/2500 [05:34<01:59,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1807/2500 [05:35<01:43,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1809/2500 [05:35<01:43,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1812/2500 [05:35<01:17,  8.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1814/2500 [05:36<02:05,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1816/2500 [05:36<02:00,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1818/2500 [05:36<01:54,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1819/2500 [05:37<02:21,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1820/2500 [05:37<03:12,  3.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1822/2500 [05:38<02:40,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1829/2500 [05:38<01:19,  8.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1831/2500 [05:39<01:37,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1834/2500 [05:39<01:18,  8.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1837/2500 [05:39<01:37,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▎  | 1839/2500 [05:40<02:23,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 1841/2500 [05:40<01:47,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 1843/2500 [05:41<01:48,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1844/2500 [05:41<01:53,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1847/2500 [05:41<01:41,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1850/2500 [05:42<01:44,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1853/2500 [05:42<01:27,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1857/2500 [05:43<01:32,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1858/2500 [05:43<01:56,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1862/2500 [05:44<01:25,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 1863/2500 [05:44<02:06,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 1866/2500 [05:44<01:48,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 1868/2500 [05:45<01:36,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 1872/2500 [05:45<01:27,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 1874/2500 [05:46<01:29,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1875/2500 [05:46<01:48,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1877/2500 [05:46<01:35,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1878/2500 [05:46<01:43,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 1883/2500 [05:47<01:26,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1885/2500 [05:47<01:19,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1886/2500 [05:48<01:41,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1888/2500 [05:48<01:51,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1891/2500 [05:48<01:26,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1893/2500 [05:49<02:24,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1896/2500 [05:50<01:54,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1899/2500 [05:50<01:38,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1901/2500 [05:50<01:31,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1906/2500 [05:51<01:04,  9.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 1908/2500 [05:51<01:10,  8.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 1910/2500 [05:51<01:15,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▋  | 1912/2500 [05:52<01:37,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1913/2500 [05:52<02:47,  3.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1917/2500 [05:53<01:59,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1919/2500 [05:54<01:50,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1924/2500 [05:54<01:03,  9.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1926/2500 [05:55<01:57,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1929/2500 [05:55<01:36,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1931/2500 [05:56<01:50,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1932/2500 [05:56<02:17,  4.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1934/2500 [05:56<02:11,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1935/2500 [05:57<01:55,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1937/2500 [05:57<01:40,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1941/2500 [05:57<01:17,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1943/2500 [05:58<01:20,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1944/2500 [05:58<01:29,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1949/2500 [05:58<01:06,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1951/2500 [05:59<01:53,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1955/2500 [06:00<01:21,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1958/2500 [06:00<01:34,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1961/2500 [06:01<01:17,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 1963/2500 [06:01<01:13,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 1966/2500 [06:02<01:25,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 1968/2500 [06:02<01:19,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1971/2500 [06:02<01:29,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1973/2500 [06:03<01:33,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1976/2500 [06:03<01:14,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1978/2500 [06:03<01:07,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1981/2500 [06:04<00:53,  9.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1983/2500 [06:05<01:42,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1987/2500 [06:05<01:35,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 1989/2500 [06:05<01:24,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 1993/2500 [06:06<01:01,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 1995/2500 [06:06<01:20,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 1997/2500 [06:07<01:11,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 1999/2500 [06:07<01:19,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2002/2500 [06:07<01:09,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2003/2500 [06:08<01:18,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2005/2500 [06:08<01:09,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2007/2500 [06:08<01:15,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2008/2500 [06:08<01:09,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2010/2500 [06:09<01:21,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2011/2500 [06:09<01:59,  4.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2013/2500 [06:10<01:45,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2019/2500 [06:10<00:55,  8.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2023/2500 [06:11<00:55,  8.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2025/2500 [06:11<00:56,  8.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2026/2500 [06:11<01:07,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2027/2500 [06:11<01:24,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2029/2500 [06:12<01:32,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2031/2500 [06:12<01:42,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2033/2500 [06:13<01:32,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2036/2500 [06:13<01:00,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2038/2500 [06:13<00:49,  9.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2040/2500 [06:13<00:55,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2042/2500 [06:14<00:54,  8.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2044/2500 [06:14<01:07,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2045/2500 [06:14<01:14,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2046/2500 [06:15<01:37,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2051/2500 [06:15<01:09,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2052/2500 [06:15<01:07,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2055/2500 [06:16<01:10,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2058/2500 [06:16<00:59,  7.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2059/2500 [06:17<01:06,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2063/2500 [06:17<00:46,  9.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2065/2500 [06:17<00:57,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2068/2500 [06:18<01:15,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2070/2500 [06:18<01:12,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2071/2500 [06:19<01:34,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2072/2500 [06:19<01:40,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2076/2500 [06:19<01:02,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2080/2500 [06:20<00:45,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2084/2500 [06:20<00:42,  9.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2086/2500 [06:21<01:28,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2087/2500 [06:22<01:45,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2092/2500 [06:22<01:16,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2097/2500 [06:23<00:56,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2099/2500 [06:23<00:47,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2101/2500 [06:23<00:47,  8.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2104/2500 [06:24<00:50,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2106/2500 [06:24<01:18,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2107/2500 [06:25<01:24,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2109/2500 [06:25<01:13,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2111/2500 [06:25<01:20,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2115/2500 [06:26<00:52,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2117/2500 [06:26<00:50,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2120/2500 [06:27<00:56,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2123/2500 [06:27<00:54,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2125/2500 [06:27<00:52,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2126/2500 [06:28<01:08,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2128/2500 [06:29<01:42,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2130/2500 [06:29<01:20,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2136/2500 [06:29<00:42,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2138/2500 [06:30<01:15,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2139/2500 [06:31<01:20,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2144/2500 [06:31<01:04,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2146/2500 [06:32<01:13,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2150/2500 [06:32<00:53,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2151/2500 [06:32<00:49,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2154/2500 [06:33<01:08,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2156/2500 [06:33<00:52,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2157/2500 [06:34<00:57,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▋ | 2161/2500 [06:34<00:41,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2162/2500 [06:34<00:55,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2167/2500 [06:35<00:52,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2169/2500 [06:36<00:57,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2170/2500 [06:36<01:00,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2171/2500 [06:36<01:04,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2173/2500 [06:37<01:04,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2176/2500 [06:37<01:05,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2177/2500 [06:37<01:05,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2180/2500 [06:38<00:50,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2182/2500 [06:38<00:45,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2184/2500 [06:38<00:47,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2185/2500 [06:39<00:57,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2187/2500 [06:39<01:01,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2188/2500 [06:39<01:01,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2190/2500 [06:40<00:59,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2193/2500 [06:40<00:34,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2195/2500 [06:40<00:41,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2197/2500 [06:41<00:50,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2200/2500 [06:41<00:44,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2202/2500 [06:41<00:35,  8.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2204/2500 [06:42<00:55,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2205/2500 [06:42<01:00,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2206/2500 [06:42<01:02,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2211/2500 [06:43<00:36,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 2214/2500 [06:43<00:38,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2215/2500 [06:43<00:40,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2216/2500 [06:44<00:50,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2218/2500 [06:44<00:48,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2220/2500 [06:44<00:47,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2222/2500 [06:45<00:51,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2225/2500 [06:46<00:56,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2226/2500 [06:46<00:54,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2231/2500 [06:46<00:38,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2233/2500 [06:47<00:31,  8.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2235/2500 [06:47<00:36,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2239/2500 [06:48<00:39,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2241/2500 [06:48<00:42,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2244/2500 [06:48<00:36,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2246/2500 [06:49<00:54,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2250/2500 [06:50<00:35,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2252/2500 [06:50<00:31,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2255/2500 [06:50<00:28,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2257/2500 [06:50<00:34,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2259/2500 [06:51<00:28,  8.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2261/2500 [06:51<00:38,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2262/2500 [06:51<00:40,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2265/2500 [06:52<00:34,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2267/2500 [06:52<00:27,  8.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2271/2500 [06:53<00:36,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2273/2500 [06:53<00:39,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2275/2500 [06:53<00:35,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2278/2500 [06:54<00:39,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2279/2500 [06:54<00:38,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2280/2500 [06:54<00:39,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2281/2500 [06:55<00:42,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2285/2500 [06:55<00:34,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2288/2500 [06:55<00:25,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2289/2500 [06:56<00:46,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2290/2500 [06:56<00:47,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2292/2500 [06:57<00:50,  4.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2298/2500 [06:57<00:25,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2300/2500 [06:58<00:21,  9.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2303/2500 [06:58<00:35,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2306/2500 [06:59<00:30,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2310/2500 [07:00<00:31,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2313/2500 [07:00<00:23,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2318/2500 [07:00<00:16, 11.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2320/2500 [07:01<00:28,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2322/2500 [07:01<00:27,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2324/2500 [07:02<00:38,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2328/2500 [07:03<00:31,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2331/2500 [07:03<00:24,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2332/2500 [07:03<00:36,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2335/2500 [07:04<00:27,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2337/2500 [07:04<00:22,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2338/2500 [07:04<00:25,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2341/2500 [07:04<00:22,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2342/2500 [07:05<00:32,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2344/2500 [07:05<00:27,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2346/2500 [07:05<00:25,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2348/2500 [07:06<00:25,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2350/2500 [07:06<00:26,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2353/2500 [07:07<00:23,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2356/2500 [07:07<00:18,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2358/2500 [07:07<00:17,  8.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2361/2500 [07:08<00:18,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2362/2500 [07:08<00:18,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2364/2500 [07:08<00:19,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2367/2500 [07:09<00:29,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2371/2500 [07:10<00:20,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2373/2500 [07:10<00:17,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2374/2500 [07:10<00:23,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2379/2500 [07:10<00:13,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2381/2500 [07:11<00:20,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2384/2500 [07:11<00:17,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2386/2500 [07:12<00:19,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2387/2500 [07:12<00:21,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2389/2500 [07:13<00:22,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2390/2500 [07:13<00:22,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2394/2500 [07:13<00:15,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2396/2500 [07:14<00:13,  7.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2399/2500 [07:14<00:09, 10.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2401/2500 [07:14<00:13,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2403/2500 [07:15<00:17,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2406/2500 [07:15<00:16,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2407/2500 [07:16<00:20,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2410/2500 [07:16<00:15,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2414/2500 [07:17<00:13,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2418/2500 [07:17<00:09,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2420/2500 [07:17<00:10,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2422/2500 [07:18<00:12,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2424/2500 [07:18<00:10,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2425/2500 [07:18<00:13,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2429/2500 [07:19<00:10,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2431/2500 [07:19<00:11,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2432/2500 [07:19<00:11,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2434/2500 [07:20<00:09,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2437/2500 [07:20<00:07,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2438/2500 [07:20<00:07,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2440/2500 [07:21<00:11,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2445/2500 [07:22<00:09,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2448/2500 [07:22<00:06,  8.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2449/2500 [07:22<00:08,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2451/2500 [07:22<00:08,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2454/2500 [07:23<00:06,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2456/2500 [07:23<00:08,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2457/2500 [07:24<00:07,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2459/2500 [07:24<00:07,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2462/2500 [07:24<00:05,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2463/2500 [07:25<00:06,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▊| 2466/2500 [07:25<00:05,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2467/2500 [07:25<00:04,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2470/2500 [07:26<00:04,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2472/2500 [07:26<00:04,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2473/2500 [07:26<00:04,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2474/2500 [07:27<00:05,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2476/2500 [07:27<00:04,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2480/2500 [07:27<00:02,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2483/2500 [07:28<00:02,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2488/2500 [07:28<00:01, 10.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2490/2500 [07:29<00:01,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2492/2500 [07:29<00:01,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2493/2500 [07:30<00:01,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2494/2500 [07:30<00:01,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2495/2500 [07:30<00:01,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2499/2500 [07:31<00:00,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 2500/2500 [07:31<00:00,  5.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 3/2500 [00:02<28:53,  1.44it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 4/2500 [00:02<20:21,  2.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 9/2500 [00:03<07:33,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 11/2500 [00:03<05:53,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 16/2500 [00:03<03:49, 10.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 18/2500 [00:03<04:52,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 20/2500 [00:05<11:36,  3.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 21/2500 [00:05<11:06,  3.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 24/2500 [00:05<08:17,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 28/2500 [00:06<07:14,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 31/2500 [00:06<05:11,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 33/2500 [00:06<04:51,  8.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 36/2500 [00:07<04:31,  9.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 39/2500 [00:07<06:44,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 43/2500 [00:08<06:41,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 49/2500 [00:09<05:42,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 51/2500 [00:09<05:44,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 57/2500 [00:10<06:03,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 59/2500 [00:11<07:12,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 61/2500 [00:11<06:17,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 63/2500 [00:12<07:14,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 65/2500 [00:12<07:48,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 68/2500 [00:12<05:46,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 71/2500 [00:13<04:51,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 74/2500 [00:13<05:47,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 77/2500 [00:14<06:14,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 78/2500 [00:14<08:41,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 81/2500 [00:15<06:21,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 82/2500 [00:15<07:02,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 86/2500 [00:15<04:15,  9.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 88/2500 [00:15<05:51,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 92/2500 [00:16<06:04,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 93/2500 [00:16<06:52,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 95/2500 [00:17<07:39,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 96/2500 [00:17<08:08,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 98/2500 [00:17<07:39,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 99/2500 [00:18<07:44,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 102/2500 [00:18<06:40,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 103/2500 [00:18<07:07,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 109/2500 [00:19<04:55,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 110/2500 [00:19<07:21,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 112/2500 [00:20<07:04,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 114/2500 [00:20<07:19,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 116/2500 [00:21<06:51,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 122/2500 [00:21<03:50, 10.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 124/2500 [00:21<05:15,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 126/2500 [00:22<06:07,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 127/2500 [00:22<07:10,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 128/2500 [00:23<10:12,  3.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 131/2500 [00:23<08:01,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 133/2500 [00:23<06:47,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 136/2500 [00:24<05:54,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 137/2500 [00:24<06:27,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 140/2500 [00:24<05:55,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 144/2500 [00:25<03:45, 10.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 146/2500 [00:25<05:06,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 149/2500 [00:26<07:04,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 150/2500 [00:26<07:07,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 152/2500 [00:26<06:22,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 155/2500 [00:26<04:44,  8.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 156/2500 [00:27<06:08,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 158/2500 [00:27<06:03,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 161/2500 [00:27<05:49,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 164/2500 [00:28<04:53,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 165/2500 [00:29<09:25,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 171/2500 [00:29<05:06,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 173/2500 [00:29<05:01,  7.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 174/2500 [00:29<05:09,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 178/2500 [00:30<05:27,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 180/2500 [00:31<07:20,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 185/2500 [00:31<06:07,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 186/2500 [00:32<06:30,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 188/2500 [00:32<06:52,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 190/2500 [00:33<08:17,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 192/2500 [00:33<07:03,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 195/2500 [00:33<05:47,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 197/2500 [00:33<04:50,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 199/2500 [00:34<05:46,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 201/2500 [00:34<05:49,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 204/2500 [00:34<05:13,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 205/2500 [00:35<05:49,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 206/2500 [00:35<06:50,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 207/2500 [00:35<07:54,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 210/2500 [00:36<06:45,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▊         | 213/2500 [00:36<05:34,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 214/2500 [00:37<09:29,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 217/2500 [00:37<08:14,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 225/2500 [00:38<03:52,  9.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 229/2500 [00:39<05:35,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 231/2500 [00:39<07:43,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 234/2500 [00:40<07:37,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 235/2500 [00:40<07:48,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 238/2500 [00:41<05:50,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 242/2500 [00:41<03:41, 10.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 246/2500 [00:41<03:32, 10.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 249/2500 [00:42<05:09,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 250/2500 [00:42<04:55,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 251/2500 [00:42<05:44,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 253/2500 [00:42<05:19,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 254/2500 [00:43<06:42,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 255/2500 [00:43<07:18,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 256/2500 [00:43<07:28,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 257/2500 [00:44<10:01,  3.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 262/2500 [00:44<05:14,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 265/2500 [00:45<05:22,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 268/2500 [00:45<03:46,  9.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 272/2500 [00:45<05:05,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 274/2500 [00:46<06:03,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 277/2500 [00:46<05:36,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 278/2500 [00:47<06:26,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 280/2500 [00:47<06:00,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 282/2500 [00:47<06:07,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 285/2500 [00:48<05:34,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 289/2500 [00:48<03:26, 10.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 291/2500 [00:48<04:29,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 293/2500 [00:49<07:05,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 295/2500 [00:49<06:18,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 298/2500 [00:50<04:51,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 300/2500 [00:50<04:36,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 303/2500 [00:50<05:58,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 304/2500 [00:51<06:23,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 308/2500 [00:51<05:30,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 311/2500 [00:51<04:10,  8.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 313/2500 [00:52<04:32,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 315/2500 [00:52<05:25,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 319/2500 [00:52<03:56,  9.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 322/2500 [00:53<05:39,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 325/2500 [00:54<06:18,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 326/2500 [00:54<06:28,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 332/2500 [00:54<03:33, 10.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 334/2500 [00:55<05:22,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 336/2500 [00:56<06:34,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 338/2500 [00:56<06:12,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 341/2500 [00:56<04:52,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 342/2500 [00:56<06:18,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 343/2500 [00:57<06:44,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 344/2500 [00:57<08:00,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 347/2500 [00:57<05:29,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 351/2500 [00:58<05:06,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 354/2500 [00:58<04:27,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 358/2500 [00:59<03:54,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 360/2500 [00:59<03:56,  9.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 364/2500 [01:00<06:24,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 365/2500 [01:00<06:16,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 367/2500 [01:00<06:20,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 372/2500 [01:01<04:40,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 374/2500 [01:01<04:47,  7.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 376/2500 [01:02<05:53,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 378/2500 [01:02<05:57,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 380/2500 [01:02<04:31,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 384/2500 [01:03<04:35,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 386/2500 [01:03<06:05,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 389/2500 [01:04<05:30,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 390/2500 [01:04<05:43,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 393/2500 [01:04<05:19,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 397/2500 [01:05<04:06,  8.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 398/2500 [01:05<06:31,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 401/2500 [01:06<05:14,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 404/2500 [01:06<04:07,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 406/2500 [01:06<05:16,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 410/2500 [01:07<05:42,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 412/2500 [01:07<04:52,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 416/2500 [01:08<04:03,  8.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 418/2500 [01:08<06:39,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 419/2500 [01:09<07:00,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 421/2500 [01:09<06:35,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 423/2500 [01:09<06:12,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 427/2500 [01:10<04:52,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 429/2500 [01:10<04:41,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 431/2500 [01:10<04:57,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 433/2500 [01:11<05:25,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 436/2500 [01:11<05:41,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 439/2500 [01:12<05:36,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 440/2500 [01:12<07:15,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 444/2500 [01:12<05:04,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 447/2500 [01:13<03:46,  9.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 449/2500 [01:13<05:41,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 452/2500 [01:13<04:10,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 454/2500 [01:14<04:49,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 456/2500 [01:14<04:29,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 457/2500 [01:15<08:05,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 460/2500 [01:15<07:48,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 464/2500 [01:16<05:24,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 468/2500 [01:16<03:14, 10.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 470/2500 [01:16<03:58,  8.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 474/2500 [01:17<03:56,  8.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 476/2500 [01:17<04:55,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 477/2500 [01:17<05:26,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 479/2500 [01:18<08:09,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 480/2500 [01:19<08:06,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 481/2500 [01:19<09:14,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 484/2500 [01:19<06:39,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 485/2500 [01:20<07:03,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 489/2500 [01:20<05:10,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 491/2500 [01:20<04:24,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 496/2500 [01:21<03:08, 10.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 498/2500 [01:21<04:44,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 499/2500 [01:22<06:48,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 502/2500 [01:22<06:02,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 505/2500 [01:22<04:11,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 507/2500 [01:23<05:48,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 510/2500 [01:23<04:41,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 513/2500 [01:24<03:55,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 515/2500 [01:24<04:07,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 517/2500 [01:25<08:06,  4.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 521/2500 [01:25<05:24,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 523/2500 [01:25<04:30,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 527/2500 [01:26<03:56,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 529/2500 [01:26<03:42,  8.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 531/2500 [01:27<06:10,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 533/2500 [01:27<06:47,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 537/2500 [01:28<05:54,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 540/2500 [01:28<04:46,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 548/2500 [01:29<04:28,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 550/2500 [01:30<04:03,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 554/2500 [01:31<05:31,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 559/2500 [01:31<04:38,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 562/2500 [01:32<04:41,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 564/2500 [01:32<04:15,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 567/2500 [01:32<04:27,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 570/2500 [01:33<06:44,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 571/2500 [01:34<07:06,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 574/2500 [01:34<05:10,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 575/2500 [01:34<05:01,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 579/2500 [01:34<04:23,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 581/2500 [01:35<04:37,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 585/2500 [01:35<04:12,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 587/2500 [01:35<03:24,  9.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 589/2500 [01:36<03:37,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 590/2500 [01:36<05:20,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 593/2500 [01:36<04:21,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 594/2500 [01:37<04:43,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 596/2500 [01:37<05:05,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 597/2500 [01:37<05:36,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 599/2500 [01:37<05:00,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 600/2500 [01:38<05:48,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 602/2500 [01:38<04:57,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 605/2500 [01:39<05:41,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 608/2500 [01:39<03:34,  8.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 610/2500 [01:39<04:34,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 612/2500 [01:40<05:02,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 614/2500 [01:40<05:12,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 618/2500 [01:41<05:26,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 620/2500 [01:41<04:59,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 623/2500 [01:42<05:27,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 625/2500 [01:42<04:40,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 627/2500 [01:42<04:24,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 629/2500 [01:42<04:09,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 632/2500 [01:43<05:09,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 634/2500 [01:43<04:48,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 636/2500 [01:43<04:51,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 639/2500 [01:44<03:32,  8.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 641/2500 [01:44<05:16,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 643/2500 [01:44<04:04,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 645/2500 [01:45<04:38,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 648/2500 [01:45<04:50,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 649/2500 [01:46<05:16,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 651/2500 [01:46<04:57,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 652/2500 [01:46<05:28,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 656/2500 [01:46<04:15,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 657/2500 [01:47<04:36,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 659/2500 [01:47<04:47,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 661/2500 [01:47<04:29,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 663/2500 [01:48<05:45,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 665/2500 [01:48<05:55,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 666/2500 [01:48<05:51,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 669/2500 [01:49<04:57,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 671/2500 [01:49<03:40,  8.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 675/2500 [01:50<04:47,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 677/2500 [01:50<04:19,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 681/2500 [01:50<04:02,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 682/2500 [01:51<03:53,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 683/2500 [01:51<04:43,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 685/2500 [01:51<04:59,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 688/2500 [01:51<03:20,  9.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 691/2500 [01:52<03:49,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 692/2500 [01:52<04:53,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 694/2500 [01:53<07:08,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 695/2500 [01:53<07:25,  4.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 697/2500 [01:54<06:56,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 700/2500 [01:54<05:20,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 705/2500 [01:54<03:31,  8.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 706/2500 [01:55<04:11,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 711/2500 [01:55<03:06,  9.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 713/2500 [01:56<06:37,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 715/2500 [01:56<05:54,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 718/2500 [01:57<05:04,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 722/2500 [01:58<04:45,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 724/2500 [01:58<04:15,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 729/2500 [01:58<03:43,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 731/2500 [01:59<03:15,  9.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 733/2500 [01:59<03:48,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 735/2500 [01:59<04:49,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 736/2500 [02:00<06:18,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 738/2500 [02:00<05:49,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 740/2500 [02:00<04:23,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 744/2500 [02:01<03:26,  8.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 746/2500 [02:01<05:07,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 750/2500 [02:02<03:57,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 754/2500 [02:02<03:41,  7.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 755/2500 [02:02<04:30,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 757/2500 [02:03<05:52,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 758/2500 [02:03<05:26,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 760/2500 [02:04<05:07,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 763/2500 [02:04<05:34,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 765/2500 [02:05<05:46,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 769/2500 [02:05<03:20,  8.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 771/2500 [02:05<04:13,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 772/2500 [02:06<05:20,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 776/2500 [02:06<05:01,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 779/2500 [02:07<04:52,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 782/2500 [02:07<04:02,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 785/2500 [02:08<03:29,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 788/2500 [02:08<03:01,  9.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 790/2500 [02:08<02:32, 11.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 792/2500 [02:09<05:49,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 794/2500 [02:09<05:29,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 795/2500 [02:09<04:56,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 796/2500 [02:10<05:55,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 799/2500 [02:10<04:45,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 800/2500 [02:10<04:29,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 802/2500 [02:11<05:05,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 809/2500 [02:11<03:11,  8.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 812/2500 [02:12<05:02,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 814/2500 [02:13<05:10,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 817/2500 [02:13<03:26,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 821/2500 [02:13<03:13,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 823/2500 [02:13<03:15,  8.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 825/2500 [02:14<03:53,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 826/2500 [02:14<04:20,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 829/2500 [02:14<03:17,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 830/2500 [02:15<07:00,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 832/2500 [02:16<06:13,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 835/2500 [02:16<03:40,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 837/2500 [02:16<03:36,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 840/2500 [02:16<03:19,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 842/2500 [02:16<03:27,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 843/2500 [02:17<04:39,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 844/2500 [02:17<05:13,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 848/2500 [02:18<03:43,  7.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 850/2500 [02:18<03:58,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 852/2500 [02:18<04:36,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 853/2500 [02:19<05:39,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 854/2500 [02:19<05:38,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 855/2500 [02:19<05:56,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 858/2500 [02:20<05:50,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 862/2500 [02:20<04:17,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 865/2500 [02:21<03:56,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 867/2500 [02:21<04:18,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 870/2500 [02:22<04:20,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 871/2500 [02:22<04:53,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 873/2500 [02:22<04:52,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 879/2500 [02:23<03:40,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 881/2500 [02:23<03:34,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 882/2500 [02:24<04:41,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 885/2500 [02:24<04:09,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 886/2500 [02:24<03:55,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 889/2500 [02:25<04:49,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 893/2500 [02:25<03:22,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 895/2500 [02:26<04:40,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 896/2500 [02:26<05:09,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 903/2500 [02:26<02:38, 10.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 905/2500 [02:28<05:26,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 907/2500 [02:28<04:40,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 909/2500 [02:28<04:51,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 910/2500 [02:28<05:29,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 916/2500 [02:29<03:43,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 918/2500 [02:29<04:04,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 921/2500 [02:30<03:07,  8.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 923/2500 [02:30<03:56,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 925/2500 [02:31<04:30,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 926/2500 [02:31<05:58,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 928/2500 [02:31<05:39,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 930/2500 [02:32<05:08,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 933/2500 [02:32<03:57,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 937/2500 [02:33<03:48,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 939/2500 [02:33<03:02,  8.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 941/2500 [02:33<03:08,  8.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 943/2500 [02:33<03:19,  7.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 946/2500 [02:34<03:19,  7.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 947/2500 [02:34<03:51,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 948/2500 [02:35<05:40,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 951/2500 [02:35<05:19,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 952/2500 [02:35<05:35,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 954/2500 [02:36<04:43,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 957/2500 [02:36<03:27,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 959/2500 [02:36<03:25,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 960/2500 [02:37<04:31,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 961/2500 [02:37<05:12,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 964/2500 [02:37<04:28,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 967/2500 [02:38<04:41,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 969/2500 [02:38<03:44,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 971/2500 [02:39<03:55,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 974/2500 [02:39<03:38,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 977/2500 [02:39<03:35,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 979/2500 [02:40<03:56,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 981/2500 [02:40<04:03,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 983/2500 [02:41<04:41,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 986/2500 [02:41<03:37,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 987/2500 [02:41<03:35,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 990/2500 [02:41<03:19,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 991/2500 [02:42<03:53,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 993/2500 [02:42<04:26,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 994/2500 [02:42<05:07,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 996/2500 [02:43<04:33,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 999/2500 [02:43<03:58,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1004/2500 [02:44<03:15,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1006/2500 [02:44<02:45,  9.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1008/2500 [02:44<03:15,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1010/2500 [02:45<03:44,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1011/2500 [02:45<03:52,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1012/2500 [02:45<06:58,  3.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1015/2500 [02:46<05:13,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1019/2500 [02:46<03:58,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1022/2500 [02:47<03:16,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1025/2500 [02:47<02:19, 10.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1027/2500 [02:47<03:51,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1029/2500 [02:48<04:52,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1033/2500 [02:48<03:22,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████▏     | 1036/2500 [02:49<03:56,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1037/2500 [02:49<04:55,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1040/2500 [02:50<04:18,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1043/2500 [02:50<02:46,  8.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1047/2500 [02:50<02:52,  8.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1049/2500 [02:51<03:14,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1051/2500 [02:51<04:32,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1053/2500 [02:52<03:50,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1054/2500 [02:52<03:47,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1056/2500 [02:52<04:15,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1058/2500 [02:53<04:39,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1060/2500 [02:53<03:38,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1063/2500 [02:53<02:37,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1065/2500 [02:53<03:30,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1066/2500 [02:54<03:51,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1069/2500 [02:54<03:23,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1072/2500 [02:54<03:15,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1073/2500 [02:55<03:40,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1077/2500 [02:56<04:41,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1079/2500 [02:56<03:57,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1082/2500 [02:57<04:31,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1084/2500 [02:57<03:55,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1086/2500 [02:57<03:02,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1088/2500 [02:57<03:22,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1091/2500 [02:58<02:53,  8.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1092/2500 [02:58<03:51,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1095/2500 [02:59<03:45,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1098/2500 [02:59<02:40,  8.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1100/2500 [02:59<03:24,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1103/2500 [03:00<03:21,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1106/2500 [03:00<03:35,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1108/2500 [03:00<03:09,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1110/2500 [03:01<03:01,  7.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1113/2500 [03:01<03:14,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1115/2500 [03:02<04:08,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1117/2500 [03:02<04:34,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1120/2500 [03:03<03:39,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1121/2500 [03:03<04:13,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1122/2500 [03:03<04:17,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1124/2500 [03:03<03:51,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1125/2500 [03:04<04:04,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1128/2500 [03:04<03:44,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1129/2500 [03:04<04:20,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1132/2500 [03:04<03:00,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1134/2500 [03:05<03:14,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1137/2500 [03:05<02:22,  9.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1139/2500 [03:05<03:09,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1141/2500 [03:06<04:34,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1144/2500 [03:07<04:02,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1145/2500 [03:07<04:08,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1150/2500 [03:07<03:07,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1152/2500 [03:08<02:47,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1154/2500 [03:08<03:12,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1156/2500 [03:08<03:33,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1159/2500 [03:09<03:28,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1162/2500 [03:09<02:58,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1164/2500 [03:09<03:23,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1165/2500 [03:10<03:20,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1166/2500 [03:10<04:13,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1168/2500 [03:10<03:35,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1171/2500 [03:11<03:25,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1173/2500 [03:11<03:34,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1174/2500 [03:11<04:03,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1175/2500 [03:11<04:15,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1177/2500 [03:12<04:47,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1180/2500 [03:12<03:36,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1182/2500 [03:13<02:42,  8.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1185/2500 [03:13<02:43,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1186/2500 [03:13<03:15,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1188/2500 [03:13<02:54,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1191/2500 [03:14<03:01,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1193/2500 [03:14<02:52,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1195/2500 [03:15<04:26,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1199/2500 [03:15<03:01,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1201/2500 [03:16<03:23,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1204/2500 [03:16<02:41,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1206/2500 [03:17<03:44,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1207/2500 [03:17<03:54,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1210/2500 [03:17<03:26,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▊     | 1213/2500 [03:17<02:40,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1214/2500 [03:18<05:03,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1216/2500 [03:18<04:16,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1221/2500 [03:19<02:46,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1223/2500 [03:19<03:36,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1226/2500 [03:20<03:10,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1229/2500 [03:20<02:30,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1231/2500 [03:20<02:33,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1232/2500 [03:21<06:18,  3.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1233/2500 [03:22<07:27,  2.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1240/2500 [03:22<02:43,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1244/2500 [03:22<02:14,  9.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1248/2500 [03:23<02:15,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1250/2500 [03:23<02:58,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1252/2500 [03:24<03:15,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1254/2500 [03:25<06:16,  3.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1259/2500 [03:26<03:10,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1261/2500 [03:26<02:53,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1268/2500 [03:26<01:47, 11.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1270/2500 [03:27<02:19,  8.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1272/2500 [03:27<02:51,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1274/2500 [03:28<04:28,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1278/2500 [03:28<03:10,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1280/2500 [03:29<03:23,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1284/2500 [03:29<02:52,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1288/2500 [03:30<02:08,  9.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1290/2500 [03:30<02:34,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1292/2500 [03:30<03:15,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1294/2500 [03:31<04:02,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1296/2500 [03:32<04:29,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1298/2500 [03:32<04:22,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1304/2500 [03:32<01:39, 12.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1306/2500 [03:33<01:50, 10.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1308/2500 [03:33<02:47,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1310/2500 [03:34<03:38,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1313/2500 [03:34<03:30,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1315/2500 [03:34<03:19,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1316/2500 [03:35<03:36,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1320/2500 [03:35<02:28,  7.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1322/2500 [03:35<02:03,  9.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1325/2500 [03:36<04:03,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1326/2500 [03:36<03:54,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1327/2500 [03:37<03:54,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1328/2500 [03:37<03:58,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1331/2500 [03:37<03:14,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1334/2500 [03:37<02:12,  8.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1337/2500 [03:38<02:52,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▎    | 1342/2500 [03:39<02:33,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1346/2500 [03:39<02:47,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1348/2500 [03:40<02:17,  8.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1350/2500 [03:40<03:32,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1351/2500 [03:40<03:43,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1352/2500 [03:41<03:54,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1356/2500 [03:41<02:43,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1358/2500 [03:42<02:46,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1360/2500 [03:42<03:06,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1361/2500 [03:42<02:53,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1364/2500 [03:43<03:08,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1367/2500 [03:43<02:09,  8.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1368/2500 [03:43<02:36,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1370/2500 [03:43<02:32,  7.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1371/2500 [03:44<03:34,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1373/2500 [03:44<03:55,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1376/2500 [03:45<03:08,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1379/2500 [03:45<02:45,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1381/2500 [03:45<02:52,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1383/2500 [03:45<02:07,  8.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1385/2500 [03:46<01:58,  9.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1387/2500 [03:46<02:00,  9.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1389/2500 [03:46<02:12,  8.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1391/2500 [03:47<02:46,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1393/2500 [03:47<03:10,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1397/2500 [03:48<03:26,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1398/2500 [03:48<03:46,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1401/2500 [03:48<02:49,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1402/2500 [03:49<03:01,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1404/2500 [03:49<03:45,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1405/2500 [03:49<03:35,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1410/2500 [03:50<03:18,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1413/2500 [03:51<02:56,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1414/2500 [03:51<03:08,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1416/2500 [03:51<02:58,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1417/2500 [03:51<03:22,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1420/2500 [03:52<02:46,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1423/2500 [03:52<01:54,  9.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1426/2500 [03:53<03:06,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1429/2500 [03:53<02:46,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1432/2500 [03:54<02:45,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1433/2500 [03:54<02:36,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1435/2500 [03:54<02:50,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1438/2500 [03:55<02:40,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1440/2500 [03:55<03:08,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1444/2500 [03:56<02:45,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1446/2500 [03:56<03:03,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1447/2500 [03:57<03:15,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1448/2500 [03:57<03:32,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1451/2500 [03:57<02:54,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1455/2500 [03:57<01:37, 10.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1457/2500 [03:59<03:48,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1459/2500 [03:59<03:32,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1461/2500 [03:59<02:59,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1464/2500 [03:59<02:29,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1466/2500 [04:00<02:21,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1471/2500 [04:01<02:37,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1473/2500 [04:01<02:23,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1477/2500 [04:02<03:17,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1479/2500 [04:02<02:36,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1481/2500 [04:02<02:33,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1484/2500 [04:03<02:59,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1489/2500 [04:04<02:24,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1491/2500 [04:04<02:32,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1495/2500 [04:05<02:11,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1496/2500 [04:05<02:08,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1498/2500 [04:05<02:29,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1501/2500 [04:06<02:41,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1502/2500 [04:06<02:28,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1503/2500 [04:06<03:05,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1505/2500 [04:07<03:38,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1507/2500 [04:07<03:18,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1508/2500 [04:07<03:19,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1511/2500 [04:08<02:52,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1514/2500 [04:08<02:15,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1516/2500 [04:08<02:02,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1519/2500 [04:09<01:52,  8.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1521/2500 [04:09<02:44,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1522/2500 [04:09<03:01,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1523/2500 [04:10<04:29,  3.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1524/2500 [04:10<04:37,  3.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1526/2500 [04:11<03:51,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1531/2500 [04:11<02:13,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1532/2500 [04:11<02:12,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 1534/2500 [04:12<03:04,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1535/2500 [04:12<02:50,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1539/2500 [04:12<01:59,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1541/2500 [04:13<02:00,  7.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1542/2500 [04:13<03:13,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1543/2500 [04:14<04:12,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1545/2500 [04:14<03:44,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1547/2500 [04:14<03:01,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1551/2500 [04:14<01:42,  9.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1553/2500 [04:15<03:01,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1554/2500 [04:16<03:14,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1559/2500 [04:16<02:35,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1561/2500 [04:16<02:26,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1562/2500 [04:17<03:12,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1566/2500 [04:17<02:17,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1567/2500 [04:17<02:20,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1569/2500 [04:18<02:30,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1572/2500 [04:18<02:12,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1575/2500 [04:19<02:31,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1577/2500 [04:19<02:24,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1578/2500 [04:19<02:32,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1582/2500 [04:20<02:07,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1583/2500 [04:20<02:13,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1584/2500 [04:20<03:08,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1585/2500 [04:21<03:23,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1587/2500 [04:21<02:54,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1588/2500 [04:21<03:11,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1592/2500 [04:22<02:13,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1593/2500 [04:22<02:29,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1595/2500 [04:22<02:20,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1599/2500 [04:22<01:37,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1601/2500 [04:23<01:45,  8.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1603/2500 [04:23<02:21,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1604/2500 [04:24<02:56,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1606/2500 [04:24<02:28,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1608/2500 [04:24<02:44,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1611/2500 [04:25<02:16,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1612/2500 [04:25<02:27,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1613/2500 [04:25<02:54,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1617/2500 [04:26<02:14,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1618/2500 [04:26<02:32,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1619/2500 [04:26<02:46,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1622/2500 [04:27<02:22,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1625/2500 [04:27<02:27,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1628/2500 [04:27<01:38,  8.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1631/2500 [04:28<02:29,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1632/2500 [04:28<02:43,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1635/2500 [04:29<02:22,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1637/2500 [04:29<02:09,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1639/2500 [04:29<02:27,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1640/2500 [04:30<02:57,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1642/2500 [04:30<03:03,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1645/2500 [04:31<02:32,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1648/2500 [04:31<01:51,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1650/2500 [04:31<02:05,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1653/2500 [04:32<02:09,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1656/2500 [04:32<02:26,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1657/2500 [04:33<02:35,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1659/2500 [04:33<03:14,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1662/2500 [04:34<02:16,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1665/2500 [04:34<01:48,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1667/2500 [04:34<01:52,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1669/2500 [04:35<02:23,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1673/2500 [04:35<01:49,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1675/2500 [04:36<03:09,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1676/2500 [04:36<03:11,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1680/2500 [04:37<02:50,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1682/2500 [04:37<03:01,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1684/2500 [04:38<02:14,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1688/2500 [04:38<01:36,  8.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1690/2500 [04:38<01:18, 10.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1692/2500 [04:38<01:20, 10.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1694/2500 [04:39<02:42,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1699/2500 [04:40<03:01,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1700/2500 [04:41<02:50,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1704/2500 [04:41<02:13,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1708/2500 [04:41<01:23,  9.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1710/2500 [04:41<01:18, 10.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1712/2500 [04:42<02:23,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 1714/2500 [04:43<02:37,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 1715/2500 [04:43<02:28,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 1716/2500 [04:43<02:35,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 1719/2500 [04:44<02:40,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1721/2500 [04:44<02:23,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1723/2500 [04:44<02:05,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1724/2500 [04:45<02:21,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 1726/2500 [04:45<02:32,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1730/2500 [04:45<01:18,  9.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1732/2500 [04:45<01:17,  9.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1734/2500 [04:46<01:46,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 1736/2500 [04:46<01:47,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 1738/2500 [04:47<02:01,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 1740/2500 [04:47<02:44,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 1742/2500 [04:48<02:48,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 1744/2500 [04:48<03:02,  4.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 1747/2500 [04:49<02:11,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1750/2500 [04:49<01:20,  9.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 1754/2500 [04:49<01:07, 11.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1756/2500 [04:49<01:52,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1758/2500 [04:50<02:13,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1760/2500 [04:50<02:26,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1764/2500 [04:51<02:04,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1766/2500 [04:52<02:04,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1768/2500 [04:52<02:12,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1770/2500 [04:52<02:03,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1773/2500 [04:52<01:24,  8.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1775/2500 [04:53<01:31,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1778/2500 [04:53<01:47,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1779/2500 [04:54<02:02,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 1782/2500 [04:54<02:06,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 1787/2500 [04:54<01:10, 10.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1789/2500 [04:55<02:34,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1791/2500 [04:56<02:20,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1793/2500 [04:56<01:43,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1797/2500 [04:56<01:20,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1799/2500 [04:56<01:16,  9.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1801/2500 [04:57<02:02,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1804/2500 [04:57<01:37,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1805/2500 [04:58<02:32,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1811/2500 [04:59<01:33,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1812/2500 [04:59<01:33,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1815/2500 [04:59<01:28,  7.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1816/2500 [04:59<02:06,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1818/2500 [05:00<02:02,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1819/2500 [05:00<02:24,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1822/2500 [05:01<02:03,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1826/2500 [05:01<01:34,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1830/2500 [05:01<01:15,  8.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1832/2500 [05:02<02:01,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1834/2500 [05:02<01:45,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1836/2500 [05:03<01:33,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 1838/2500 [05:03<02:03,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▎  | 1842/2500 [05:04<01:32,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1845/2500 [05:04<01:07,  9.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1847/2500 [05:04<01:10,  9.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1849/2500 [05:04<01:20,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1851/2500 [05:05<01:18,  8.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1853/2500 [05:05<02:10,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1855/2500 [05:06<01:51,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1857/2500 [05:06<01:51,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1858/2500 [05:07<02:43,  3.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 1863/2500 [05:07<01:36,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 1865/2500 [05:07<01:17,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 1867/2500 [05:07<01:26,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 1870/2500 [05:08<01:37,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 1875/2500 [05:08<01:00, 10.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1877/2500 [05:09<02:19,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1879/2500 [05:10<02:23,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1880/2500 [05:10<02:21,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1883/2500 [05:10<01:56,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1888/2500 [05:11<01:27,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1891/2500 [05:11<01:04,  9.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1894/2500 [05:12<01:32,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1895/2500 [05:12<01:43,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1897/2500 [05:12<01:42,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1898/2500 [05:13<01:49,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1900/2500 [05:13<01:32,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1902/2500 [05:13<01:42,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1904/2500 [05:14<02:00,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 1909/2500 [05:14<01:04,  9.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 1911/2500 [05:15<01:39,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1913/2500 [05:15<01:45,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1914/2500 [05:15<01:59,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1917/2500 [05:16<01:36,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1919/2500 [05:16<01:14,  7.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1922/2500 [05:16<01:00,  9.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1924/2500 [05:17<01:21,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1926/2500 [05:17<02:02,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1929/2500 [05:18<01:51,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1931/2500 [05:18<01:29,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1933/2500 [05:19<01:31,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1937/2500 [05:19<01:17,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1938/2500 [05:19<01:23,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1940/2500 [05:20<01:40,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1942/2500 [05:20<01:27,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1944/2500 [05:20<01:17,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1945/2500 [05:21<01:49,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1948/2500 [05:21<01:23,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1951/2500 [05:22<01:33,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1955/2500 [05:22<01:22,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1958/2500 [05:22<00:59,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1960/2500 [05:23<01:14,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1961/2500 [05:23<01:31,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 1964/2500 [05:23<01:17,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 1965/2500 [05:24<01:45,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 1968/2500 [05:24<01:27,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1970/2500 [05:25<01:33,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1972/2500 [05:25<01:18,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1974/2500 [05:25<01:01,  8.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1976/2500 [05:25<01:14,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1977/2500 [05:26<01:18,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1980/2500 [05:26<01:29,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1983/2500 [05:27<01:28,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1984/2500 [05:27<01:32,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1985/2500 [05:27<01:47,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1986/2500 [05:27<01:49,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 1989/2500 [05:28<01:27,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 1991/2500 [05:28<01:09,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 1995/2500 [05:28<00:52,  9.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 1996/2500 [05:29<01:29,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 1997/2500 [05:29<01:32,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2000/2500 [05:30<01:22,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2004/2500 [05:31<01:35,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2008/2500 [05:31<00:56,  8.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2010/2500 [05:31<01:32,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2013/2500 [05:32<01:29,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2017/2500 [05:32<01:02,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2019/2500 [05:33<00:59,  8.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2021/2500 [05:33<00:53,  8.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2024/2500 [05:33<01:00,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2026/2500 [05:34<01:14,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2027/2500 [05:34<01:38,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2028/2500 [05:34<01:39,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2031/2500 [05:35<01:36,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2032/2500 [05:35<01:37,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2035/2500 [05:35<01:08,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2037/2500 [05:36<01:10,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2039/2500 [05:36<01:05,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2041/2500 [05:36<00:53,  8.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2044/2500 [05:36<00:55,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2046/2500 [05:37<01:06,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2050/2500 [05:37<01:15,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2052/2500 [05:38<01:09,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2054/2500 [05:38<01:23,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2056/2500 [05:39<01:21,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2057/2500 [05:39<01:39,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2060/2500 [05:40<01:32,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2062/2500 [05:40<01:04,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2064/2500 [05:40<00:53,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2067/2500 [05:40<00:53,  8.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2071/2500 [05:41<00:45,  9.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2073/2500 [05:41<01:06,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2079/2500 [05:42<00:52,  8.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2081/2500 [05:42<00:54,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2083/2500 [05:43<01:42,  4.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2085/2500 [05:44<01:26,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2088/2500 [05:44<01:12,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2090/2500 [05:44<01:00,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2092/2500 [05:44<01:02,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2094/2500 [05:45<00:49,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2096/2500 [05:45<01:01,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2100/2500 [05:45<00:45,  8.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2102/2500 [05:46<01:11,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2105/2500 [05:47<01:09,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2107/2500 [05:47<01:01,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2108/2500 [05:47<01:12,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2111/2500 [05:48<01:03,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2112/2500 [05:48<01:05,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2115/2500 [05:48<01:03,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2119/2500 [05:49<00:50,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2121/2500 [05:49<00:55,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2123/2500 [05:49<01:00,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2126/2500 [05:50<01:09,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2127/2500 [05:50<01:18,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2128/2500 [05:51<01:19,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2132/2500 [05:51<00:49,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2133/2500 [05:51<00:52,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2137/2500 [05:52<00:44,  8.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2139/2500 [05:52<01:04,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2140/2500 [05:52<01:06,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2142/2500 [05:53<01:06,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2143/2500 [05:53<00:59,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2144/2500 [05:53<01:10,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2147/2500 [05:54<00:57,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2152/2500 [05:54<00:37,  9.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2154/2500 [05:54<00:33, 10.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▋ | 2158/2500 [05:55<00:52,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2160/2500 [05:55<00:42,  7.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2162/2500 [05:56<00:52,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2163/2500 [05:56<01:05,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2164/2500 [05:56<01:06,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2166/2500 [05:57<01:00,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2169/2500 [05:57<00:51,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2171/2500 [05:57<00:52,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2173/2500 [05:58<00:57,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2177/2500 [05:58<00:45,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2179/2500 [05:58<00:35,  9.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2181/2500 [05:59<00:58,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2184/2500 [06:00<00:53,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2186/2500 [06:00<00:51,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2190/2500 [06:00<00:41,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2191/2500 [06:01<00:51,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2193/2500 [06:01<00:43,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2195/2500 [06:01<00:40,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2197/2500 [06:01<00:42,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2198/2500 [06:02<01:03,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2201/2500 [06:02<00:48,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2203/2500 [06:03<00:53,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2209/2500 [06:03<00:32,  8.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2211/2500 [06:03<00:28, 10.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2213/2500 [06:04<00:32,  8.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 2216/2500 [06:04<00:46,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2217/2500 [06:05<01:04,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2220/2500 [06:05<00:51,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2224/2500 [06:06<00:39,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2226/2500 [06:06<00:46,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2229/2500 [06:06<00:29,  9.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2231/2500 [06:07<00:31,  8.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2233/2500 [06:08<01:00,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2235/2500 [06:08<01:06,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2241/2500 [06:09<00:39,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2243/2500 [06:09<00:37,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2244/2500 [06:10<00:50,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2246/2500 [06:10<00:43,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2247/2500 [06:10<00:46,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2252/2500 [06:10<00:34,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2254/2500 [06:11<00:27,  8.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2256/2500 [06:11<00:31,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2258/2500 [06:12<00:55,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2259/2500 [06:12<00:58,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2260/2500 [06:13<01:04,  3.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2263/2500 [06:13<00:48,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2265/2500 [06:13<00:47,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2271/2500 [06:14<00:21, 10.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2273/2500 [06:14<00:36,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2277/2500 [06:15<00:40,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2279/2500 [06:16<00:38,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2280/2500 [06:16<00:42,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2283/2500 [06:17<00:45,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2285/2500 [06:17<00:44,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2290/2500 [06:18<00:28,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2291/2500 [06:18<00:42,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2292/2500 [06:18<00:43,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2293/2500 [06:19<00:57,  3.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2296/2500 [06:19<00:42,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2298/2500 [06:20<00:55,  3.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2300/2500 [06:20<00:36,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2304/2500 [06:21<00:25,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2307/2500 [06:21<00:27,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2308/2500 [06:22<00:37,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2313/2500 [06:22<00:23,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2314/2500 [06:22<00:22,  8.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2316/2500 [06:22<00:25,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2318/2500 [06:23<00:25,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2319/2500 [06:23<00:27,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2320/2500 [06:23<00:37,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2322/2500 [06:24<00:35,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2326/2500 [06:24<00:24,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2327/2500 [06:24<00:29,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2330/2500 [06:25<00:28,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2332/2500 [06:25<00:28,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2335/2500 [06:26<00:21,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2336/2500 [06:26<00:20,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2337/2500 [06:26<00:27,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2339/2500 [06:26<00:26,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▎| 2341/2500 [06:27<00:29,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2343/2500 [06:27<00:30,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2344/2500 [06:27<00:31,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2346/2500 [06:28<00:26,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2348/2500 [06:28<00:30,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2350/2500 [06:28<00:22,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2352/2500 [06:29<00:26,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2355/2500 [06:29<00:23,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2359/2500 [06:30<00:22,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2361/2500 [06:30<00:17,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2363/2500 [06:31<00:21,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2365/2500 [06:31<00:31,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2369/2500 [06:32<00:19,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2371/2500 [06:32<00:26,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2372/2500 [06:33<00:26,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2375/2500 [06:33<00:23,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2378/2500 [06:33<00:15,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2380/2500 [06:34<00:21,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2383/2500 [06:34<00:16,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2385/2500 [06:35<00:19,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2386/2500 [06:35<00:24,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2389/2500 [06:35<00:21,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2393/2500 [06:36<00:13,  7.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2395/2500 [06:36<00:13,  8.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2398/2500 [06:36<00:10,  9.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2400/2500 [06:37<00:17,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2401/2500 [06:38<00:26,  3.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2404/2500 [06:38<00:20,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2406/2500 [06:38<00:17,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2407/2500 [06:39<00:19,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2409/2500 [06:39<00:18,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▋| 2411/2500 [06:40<00:21,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2413/2500 [06:40<00:16,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2416/2500 [06:40<00:13,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2418/2500 [06:40<00:10,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2421/2500 [06:41<00:10,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2423/2500 [06:41<00:14,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2426/2500 [06:42<00:11,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2427/2500 [06:42<00:18,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2430/2500 [06:43<00:14,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2432/2500 [06:43<00:12,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2435/2500 [06:44<00:09,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2439/2500 [06:44<00:06, 10.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2441/2500 [06:44<00:09,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2444/2500 [06:45<00:10,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2447/2500 [06:46<00:09,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2450/2500 [06:46<00:09,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2451/2500 [06:46<00:09,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2454/2500 [06:47<00:07,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2458/2500 [06:47<00:06,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2461/2500 [06:48<00:05,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2463/2500 [06:48<00:04,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2465/2500 [06:48<00:03,  8.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▊| 2468/2500 [06:49<00:06,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2470/2500 [06:50<00:06,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2471/2500 [06:50<00:06,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2474/2500 [06:50<00:04,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2478/2500 [06:51<00:03,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2480/2500 [06:51<00:02,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2483/2500 [06:52<00:02,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2485/2500 [06:52<00:03,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2486/2500 [06:52<00:02,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2487/2500 [06:53<00:03,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2490/2500 [06:53<00:01,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2492/2500 [06:53<00:01,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2493/2500 [06:54<00:01,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2494/2500 [06:54<00:01,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2496/2500 [06:54<00:00,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2499/2500 [06:55<00:00,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 2500/2500 [06:57<00:00,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "Evaluation metrics:  {'f1': 0.0, 'em': 0.0, 'acc': 0.22573333333333334}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# obtain SEWOptimizer after having more roles, default\n",
    "optimizer = SEWOptimizer(\n",
    "    graph=sew_graph, \n",
    "    evaluator=evaluator, \n",
    "    llm=llm, \n",
    "    max_steps=20,\n",
    "    eval_rounds=3, \n",
    "    repr_scheme=\"python\", \n",
    "    optimize_mode=\"all\", \n",
    "    order=\"zero-order\",\n",
    "    max_rounds=20,\n",
    ")\n",
    "\n",
    "with suppress_logger_info():\n",
    "    metrics = optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "print(\"Evaluation metrics: \", metrics)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "44ab2303",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'f1': 0.0, 'em': 0.0, 'acc': 0.22573333333333334}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8b05058e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 16:52:10.868\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m678\u001b[0m - \u001b[1mOptimizing the SEWWorkFlowGraph workflow with python representation.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:10.869\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m682\u001b[0m - \u001b[1mRun initial evaluation on the original workflow ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|▏         | 1/80 [00:02<03:02,  2.31s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▎         | 2/80 [00:02<01:23,  1.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 7/80 [00:02<00:17,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 9/80 [00:03<00:12,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 14/80 [00:03<00:08,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 17/80 [00:03<00:05, 11.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 19/80 [00:03<00:06,  9.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 25/80 [00:05<00:09,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 27/80 [00:05<00:07,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 29/80 [00:06<00:08,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 31/80 [00:06<00:07,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 38/80 [00:06<00:04,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 40/80 [00:07<00:05,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▎    | 42/80 [00:08<00:07,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 44/80 [00:08<00:06,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 47/80 [00:09<00:05,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 49/80 [00:09<00:05,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 52/80 [00:09<00:02,  9.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 57/80 [00:09<00:02, 11.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 59/80 [00:10<00:02,  9.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 61/80 [00:11<00:04,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 63/80 [00:11<00:03,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 64/80 [00:11<00:03,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 65/80 [00:12<00:03,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 68/80 [00:12<00:02,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 70/80 [00:13<00:01,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 73/80 [00:13<00:00,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 76/80 [00:13<00:00, 10.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 80/80 [00:13<00:00,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-27 16:52:25.006\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m685\u001b[0m - \u001b[1mInitial metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.225}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 16:52:26.275\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.423 | Total tokens: 5976787 | Current cost: $0.000 | Current tokens: 117\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:27.748\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.423 | Total tokens: 5976928 | Current cost: $0.000 | Current tokens: 141\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:27.749\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 0: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:31.088\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.423 | Total tokens: 5977170 | Current cost: $0.000 | Current tokens: 242\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:31.677\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5977475 | Current cost: $0.000 | Current tokens: 305\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:31.678\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 1: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:32.907\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5977557 | Current cost: $0.000 | Current tokens: 82\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:33.767\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5977698 | Current cost: $0.000 | Current tokens: 141\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:33.768\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 2: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:34.532\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5977810 | Current cost: $0.000 | Current tokens: 112\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:36.176\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5978009 | Current cost: $0.000 | Current tokens: 199\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:36.177\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:36.177\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 3: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:37.341\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5978116 | Current cost: $0.000 | Current tokens: 107\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:37.992\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5978286 | Current cost: $0.000 | Current tokens: 170\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:37.993\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 4: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:39.154\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5978364 | Current cost: $0.000 | Current tokens: 78\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:40.681\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5978515 | Current cost: $0.000 | Current tokens: 151\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:40.681\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:40.682\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 5: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:41.880\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5978624 | Current cost: $0.000 | Current tokens: 109\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:43.916\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5978863 | Current cost: $0.000 | Current tokens: 239\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:43.916\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:43.917\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 6: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:44.938\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5978925 | Current cost: $0.000 | Current tokens: 62\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:45.689\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5979051 | Current cost: $0.000 | Current tokens: 126\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:45.690\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 7: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:46.228\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5979136 | Current cost: $0.000 | Current tokens: 85\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:47.054\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5979270 | Current cost: $0.000 | Current tokens: 134\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:47.055\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 8: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:47.733\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5979332 | Current cost: $0.000 | Current tokens: 62\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:48.746\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5979458 | Current cost: $0.000 | Current tokens: 126\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:48.747\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 9: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:49.475\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5979552 | Current cost: $0.000 | Current tokens: 94\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:50.725\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5979687 | Current cost: $0.000 | Current tokens: 135\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:50.726\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 10: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:51.155\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5979753 | Current cost: $0.000 | Current tokens: 66\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2025-12-27 16:52:52.458\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5979879 | Current cost: $0.000 | Current tokens: 126\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:52.459\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 11: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:54.084\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5980021 | Current cost: $0.000 | Current tokens: 142\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:55.366\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5980248 | Current cost: $0.000 | Current tokens: 227\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:55.367\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:55.367\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 12: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:57.096\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5980357 | Current cost: $0.000 | Current tokens: 109\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:57.782\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5980534 | Current cost: $0.000 | Current tokens: 177\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:57.783\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 13: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:58.294\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5980602 | Current cost: $0.000 | Current tokens: 68\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:59.071\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5980728 | Current cost: $0.000 | Current tokens: 126\u001b[0m\n",
      "\u001b[32m2025-12-27 16:52:59.072\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 14: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:02.791\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.424 | Total tokens: 5980938 | Current cost: $0.000 | Current tokens: 210\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:03.500\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.425 | Total tokens: 5981216 | Current cost: $0.000 | Current tokens: 278\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:03.501\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 15: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:04.174\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.425 | Total tokens: 5981330 | Current cost: $0.000 | Current tokens: 114\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:05.103\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.425 | Total tokens: 5981474 | Current cost: $0.000 | Current tokens: 144\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:05.103\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:05.104\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 16: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:05.777\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.425 | Total tokens: 5981552 | Current cost: $0.000 | Current tokens: 78\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:07.262\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.425 | Total tokens: 5981701 | Current cost: $0.000 | Current tokens: 149\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:07.263\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:07.263\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 17: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:10.276\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.425 | Total tokens: 5981857 | Current cost: $0.000 | Current tokens: 156\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:11.687\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.425 | Total tokens: 5982094 | Current cost: $0.000 | Current tokens: 237\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:11.687\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:11.688\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 18: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:19.376\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.425 | Total tokens: 5982611 | Current cost: $0.000 | Current tokens: 517\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:20.217\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $1.425 | Total tokens: 5983177 | Current cost: $0.000 | Current tokens: 566\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:20.218\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 19: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:20.218\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m707\u001b[0m - \u001b[1mReach the maximum number of steps 20. Stop the optimization.\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:20.219\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m710\u001b[0m - \u001b[1mRestore the best graph from the snapshot ...\u001b[0m\n",
      "\u001b[32m2025-12-27 16:53:20.220\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mrestore_best_graph\u001b[0m:\u001b[36m814\u001b[0m - \u001b[1mRestore the best graph from snapshot with metrics {'f1': 0.0, 'em': 0.0, 'acc': 0.225} ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 1/2500 [00:02<1:40:05,  2.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 3/2500 [00:02<28:30,  1.46it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 6/2500 [00:03<12:18,  3.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 10/2500 [00:03<05:46,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 13/2500 [00:03<05:26,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 15/2500 [00:04<05:30,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 17/2500 [00:04<05:17,  7.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 19/2500 [00:04<05:25,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 21/2500 [00:05<11:07,  3.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 23/2500 [00:05<07:53,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 26/2500 [00:06<06:33,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 27/2500 [00:06<07:42,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 28/2500 [00:06<09:05,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 29/2500 [00:07<08:58,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 33/2500 [00:07<05:18,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 34/2500 [00:07<06:19,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 38/2500 [00:08<05:53,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 39/2500 [00:08<10:11,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 40/2500 [00:09<10:51,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 44/2500 [00:09<06:41,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 45/2500 [00:09<07:07,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 48/2500 [00:10<06:01,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 50/2500 [00:10<05:51,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 53/2500 [00:10<04:35,  8.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 56/2500 [00:11<08:19,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 57/2500 [00:11<07:37,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 59/2500 [00:12<07:43,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 60/2500 [00:12<08:11,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 63/2500 [00:13<07:26,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 67/2500 [00:13<05:58,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 69/2500 [00:13<05:49,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 71/2500 [00:14<05:58,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 72/2500 [00:14<08:25,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 74/2500 [00:15<08:44,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 77/2500 [00:15<08:46,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 80/2500 [00:16<06:48,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 83/2500 [00:16<06:29,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 85/2500 [00:16<05:58,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 87/2500 [00:17<05:03,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 88/2500 [00:17<08:35,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 93/2500 [00:18<06:15,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 94/2500 [00:18<07:50,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 95/2500 [00:19<11:51,  3.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 97/2500 [00:19<10:08,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 98/2500 [00:20<10:20,  3.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 105/2500 [00:20<04:42,  8.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 107/2500 [00:21<06:31,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 108/2500 [00:21<06:15,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 109/2500 [00:21<07:04,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 111/2500 [00:21<07:25,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 112/2500 [00:22<11:11,  3.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 115/2500 [00:22<07:55,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 122/2500 [00:23<03:17, 12.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 125/2500 [00:23<04:20,  9.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 127/2500 [00:23<04:35,  8.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 129/2500 [00:24<07:17,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 131/2500 [00:25<12:08,  3.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 135/2500 [00:26<08:07,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 138/2500 [00:26<05:43,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 140/2500 [00:26<05:06,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 147/2500 [00:27<04:06,  9.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 150/2500 [00:28<08:20,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 152/2500 [00:29<07:27,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 154/2500 [00:29<07:13,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 157/2500 [00:29<07:02,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 159/2500 [00:30<05:42,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 163/2500 [00:30<04:27,  8.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 165/2500 [00:31<07:48,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 167/2500 [00:31<06:39,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 169/2500 [00:31<05:28,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 171/2500 [00:32<07:10,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 173/2500 [00:32<08:38,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 175/2500 [00:33<07:36,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 177/2500 [00:33<07:39,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 179/2500 [00:34<09:00,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 182/2500 [00:34<05:35,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 184/2500 [00:34<07:27,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 186/2500 [00:35<07:45,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 187/2500 [00:35<07:55,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 190/2500 [00:36<06:27,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 194/2500 [00:36<04:45,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 196/2500 [00:36<03:58,  9.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 198/2500 [00:36<03:47, 10.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 200/2500 [00:37<06:40,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 201/2500 [00:37<08:57,  4.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 202/2500 [00:38<10:03,  3.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 205/2500 [00:38<08:18,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 209/2500 [00:39<05:46,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▊         | 213/2500 [00:39<04:29,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 217/2500 [00:39<02:53, 13.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 219/2500 [00:41<08:40,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 221/2500 [00:41<07:59,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 223/2500 [00:41<07:08,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 225/2500 [00:42<09:06,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 230/2500 [00:42<05:04,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 232/2500 [00:42<04:33,  8.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 234/2500 [00:43<05:07,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 236/2500 [00:43<07:06,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 237/2500 [00:44<08:07,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 240/2500 [00:44<06:49,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 242/2500 [00:44<05:09,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 244/2500 [00:45<08:09,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 250/2500 [00:45<03:58,  9.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 252/2500 [00:46<04:16,  8.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 254/2500 [00:46<05:33,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 256/2500 [00:46<05:12,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 259/2500 [00:47<08:07,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 261/2500 [00:48<07:07,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 262/2500 [00:48<08:05,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 265/2500 [00:49<06:52,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 269/2500 [00:49<04:28,  8.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 271/2500 [00:49<06:20,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 273/2500 [00:50<07:46,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 274/2500 [00:50<07:06,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 277/2500 [00:51<06:03,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 278/2500 [00:51<08:44,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 279/2500 [00:51<09:23,  3.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 280/2500 [00:52<09:27,  3.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 282/2500 [00:52<07:32,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 285/2500 [00:52<05:31,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 286/2500 [00:53<08:23,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 289/2500 [00:53<06:28,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 292/2500 [00:53<04:48,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 294/2500 [00:54<05:15,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 297/2500 [00:54<04:06,  8.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 298/2500 [00:54<04:24,  8.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 300/2500 [00:55<09:44,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 305/2500 [00:55<05:01,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 307/2500 [00:56<05:05,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 309/2500 [00:56<05:11,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 314/2500 [00:57<04:39,  7.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 317/2500 [00:57<05:04,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 319/2500 [00:58<06:19,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 322/2500 [00:58<05:28,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 324/2500 [00:58<05:06,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 326/2500 [00:59<05:10,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 327/2500 [00:59<06:00,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 330/2500 [00:59<04:59,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 331/2500 [01:00<06:35,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 332/2500 [01:00<08:34,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 333/2500 [01:00<08:35,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 339/2500 [01:01<04:29,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 341/2500 [01:01<05:37,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 343/2500 [01:02<07:28,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 345/2500 [01:02<05:25,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 348/2500 [01:03<06:20,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 351/2500 [01:03<04:20,  8.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 353/2500 [01:03<04:30,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 355/2500 [01:04<06:38,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 358/2500 [01:04<04:38,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 360/2500 [01:04<04:41,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 362/2500 [01:05<06:02,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 363/2500 [01:05<06:32,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 364/2500 [01:05<07:29,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 365/2500 [01:06<08:12,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 367/2500 [01:06<07:11,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 372/2500 [01:07<04:56,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 373/2500 [01:07<05:07,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 376/2500 [01:07<05:22,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 377/2500 [01:07<05:30,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 380/2500 [01:08<05:45,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 381/2500 [01:08<07:19,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 382/2500 [01:09<09:27,  3.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 383/2500 [01:09<09:52,  3.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 386/2500 [01:09<06:36,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 388/2500 [01:10<05:17,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 392/2500 [01:10<04:57,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 394/2500 [01:10<04:40,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 395/2500 [01:11<06:45,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 396/2500 [01:11<11:08,  3.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 398/2500 [01:12<09:34,  3.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 402/2500 [01:12<06:09,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 407/2500 [01:13<04:28,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 409/2500 [01:13<04:10,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 411/2500 [01:14<06:25,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 414/2500 [01:14<05:14,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 415/2500 [01:14<05:35,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 416/2500 [01:15<06:45,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 419/2500 [01:15<06:33,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 421/2500 [01:16<05:56,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 428/2500 [01:16<03:10, 10.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 430/2500 [01:17<04:50,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 432/2500 [01:17<04:18,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 434/2500 [01:17<05:45,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 436/2500 [01:18<06:46,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 439/2500 [01:19<08:11,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 444/2500 [01:19<04:52,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 447/2500 [01:20<05:22,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 449/2500 [01:20<04:25,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 451/2500 [01:21<06:23,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 453/2500 [01:21<05:39,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 455/2500 [01:22<07:12,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 456/2500 [01:22<06:44,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 457/2500 [01:22<07:52,  4.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 460/2500 [01:22<05:52,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 462/2500 [01:23<05:13,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 464/2500 [01:23<05:04,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 470/2500 [01:23<02:44, 12.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 472/2500 [01:24<06:09,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 475/2500 [01:25<05:54,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 477/2500 [01:25<05:25,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 478/2500 [01:25<06:33,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 480/2500 [01:26<05:56,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 481/2500 [01:26<07:15,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 483/2500 [01:26<07:32,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 486/2500 [01:27<05:30,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 489/2500 [01:27<03:33,  9.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 491/2500 [01:27<04:22,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 493/2500 [01:28<04:05,  8.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 495/2500 [01:29<08:53,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 496/2500 [01:29<09:30,  3.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 500/2500 [01:29<06:26,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 501/2500 [01:30<06:58,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 503/2500 [01:30<07:07,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 507/2500 [01:31<05:49,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 510/2500 [01:31<04:01,  8.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 514/2500 [01:31<03:59,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 515/2500 [01:32<06:43,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 517/2500 [01:33<08:02,  4.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 522/2500 [01:33<05:14,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 524/2500 [01:34<05:18,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 526/2500 [01:34<03:56,  8.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 530/2500 [01:34<03:29,  9.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 532/2500 [01:35<04:59,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 534/2500 [01:35<06:34,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 537/2500 [01:36<05:30,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 541/2500 [01:36<04:28,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 543/2500 [01:36<03:40,  8.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 546/2500 [01:37<05:20,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 548/2500 [01:38<06:51,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 550/2500 [01:38<06:15,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 551/2500 [01:38<05:46,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 552/2500 [01:39<07:30,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 555/2500 [01:39<06:40,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 559/2500 [01:40<03:42,  8.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 561/2500 [01:40<04:24,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 564/2500 [01:40<04:23,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 565/2500 [01:41<05:20,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 566/2500 [01:41<07:04,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 568/2500 [01:41<06:35,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 570/2500 [01:42<07:40,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 572/2500 [01:42<05:33,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 574/2500 [01:42<04:34,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 578/2500 [01:43<04:46,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 580/2500 [01:43<05:17,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 583/2500 [01:44<04:15,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 585/2500 [01:44<04:17,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 588/2500 [01:45<05:28,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 589/2500 [01:45<06:13,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 591/2500 [01:45<06:11,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 592/2500 [01:46<07:07,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 593/2500 [01:46<08:21,  3.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 598/2500 [01:47<05:11,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 601/2500 [01:47<05:21,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 605/2500 [01:47<03:55,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 606/2500 [01:48<04:36,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 608/2500 [01:48<05:07,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 609/2500 [01:49<07:04,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 612/2500 [01:49<05:43,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 614/2500 [01:50<05:54,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 617/2500 [01:50<05:50,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 620/2500 [01:51<05:23,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 623/2500 [01:51<04:43,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 625/2500 [01:51<04:07,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 629/2500 [01:52<04:06,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 631/2500 [01:52<06:18,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 632/2500 [01:53<08:36,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 633/2500 [01:53<08:16,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 635/2500 [01:54<07:09,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 638/2500 [01:54<05:40,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 642/2500 [01:54<04:05,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 645/2500 [01:55<03:28,  8.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 647/2500 [01:55<04:33,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 648/2500 [01:55<04:38,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 650/2500 [01:56<05:00,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 652/2500 [01:56<06:41,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 653/2500 [01:56<05:56,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 655/2500 [01:57<05:53,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 658/2500 [01:57<05:57,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 659/2500 [01:58<06:24,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 662/2500 [01:58<05:10,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 664/2500 [01:58<05:19,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 667/2500 [01:59<04:21,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 669/2500 [01:59<04:06,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 672/2500 [02:00<05:20,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 674/2500 [02:00<04:10,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 676/2500 [02:00<04:35,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 678/2500 [02:01<03:52,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 681/2500 [02:01<05:06,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 683/2500 [02:02<05:10,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 686/2500 [02:02<04:00,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 687/2500 [02:02<05:11,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 688/2500 [02:02<05:31,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 689/2500 [02:03<05:53,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 692/2500 [02:03<05:43,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 695/2500 [02:04<04:31,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 698/2500 [02:04<03:27,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 699/2500 [02:04<04:01,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 700/2500 [02:04<04:57,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 704/2500 [02:05<04:59,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 706/2500 [02:05<04:13,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 707/2500 [02:05<04:13,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 708/2500 [02:06<04:52,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 709/2500 [02:06<06:12,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 715/2500 [02:07<03:53,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 717/2500 [02:07<03:22,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 720/2500 [02:07<04:00,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 723/2500 [02:08<04:32,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 725/2500 [02:08<05:20,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 728/2500 [02:09<06:36,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 730/2500 [02:09<04:37,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 732/2500 [02:10<04:18,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 735/2500 [02:10<03:50,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 738/2500 [02:11<04:27,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 740/2500 [02:11<04:01,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 742/2500 [02:12<06:12,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 744/2500 [02:12<04:22,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 745/2500 [02:12<05:53,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 747/2500 [02:12<05:10,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 748/2500 [02:13<06:05,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 751/2500 [02:13<04:36,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 753/2500 [02:14<05:57,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 754/2500 [02:14<05:40,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 758/2500 [02:14<03:39,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 761/2500 [02:14<03:19,  8.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 763/2500 [02:15<03:56,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 764/2500 [02:15<06:27,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 768/2500 [02:16<05:30,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 770/2500 [02:16<05:11,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 772/2500 [02:17<04:08,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 778/2500 [02:17<03:05,  9.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 780/2500 [02:18<04:09,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 783/2500 [02:18<03:32,  8.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 784/2500 [02:19<06:40,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 786/2500 [02:19<06:01,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 789/2500 [02:20<05:15,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 792/2500 [02:20<03:46,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 793/2500 [02:20<04:52,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 800/2500 [02:21<04:03,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 803/2500 [02:21<03:32,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 807/2500 [02:23<05:38,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 808/2500 [02:23<06:51,  4.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 812/2500 [02:24<04:58,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 813/2500 [02:24<05:28,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 816/2500 [02:24<04:32,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 819/2500 [02:25<04:12,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 820/2500 [02:25<04:36,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 822/2500 [02:25<04:01,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 823/2500 [02:26<04:57,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 824/2500 [02:26<05:29,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 827/2500 [02:26<04:17,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 830/2500 [02:26<03:11,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 833/2500 [02:27<04:00,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 836/2500 [02:28<04:13,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 839/2500 [02:28<03:39,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 843/2500 [02:28<03:10,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 845/2500 [02:29<05:53,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 846/2500 [02:30<06:50,  4.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 849/2500 [02:30<06:06,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 850/2500 [02:30<05:50,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 853/2500 [02:31<05:18,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 858/2500 [02:31<03:01,  9.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 860/2500 [02:31<02:59,  9.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 861/2500 [02:32<03:48,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 862/2500 [02:32<04:51,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 863/2500 [02:33<07:19,  3.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 865/2500 [02:33<05:54,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 867/2500 [02:33<04:34,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 869/2500 [02:33<03:13,  8.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 873/2500 [02:34<02:55,  9.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 875/2500 [02:34<03:29,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 876/2500 [02:34<04:08,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 878/2500 [02:35<04:33,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 882/2500 [02:35<03:04,  8.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 884/2500 [02:36<04:27,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 886/2500 [02:36<04:08,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 888/2500 [02:36<04:01,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 890/2500 [02:37<04:13,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 893/2500 [02:37<03:17,  8.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 894/2500 [02:37<03:41,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 895/2500 [02:38<07:55,  3.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 896/2500 [02:38<07:35,  3.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 899/2500 [02:38<05:10,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 900/2500 [02:39<04:50,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 901/2500 [02:39<05:20,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 903/2500 [02:39<04:25,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 904/2500 [02:39<05:37,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 906/2500 [02:40<05:04,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 910/2500 [02:40<02:54,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 912/2500 [02:40<02:50,  9.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 915/2500 [02:41<04:01,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 917/2500 [02:41<04:56,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 918/2500 [02:42<05:09,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 920/2500 [02:42<04:08,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 921/2500 [02:42<05:18,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 923/2500 [02:43<04:56,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 924/2500 [02:43<05:31,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 926/2500 [02:43<04:53,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 927/2500 [02:44<06:27,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 928/2500 [02:44<06:53,  3.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 931/2500 [02:44<04:55,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 934/2500 [02:45<03:45,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 935/2500 [02:45<04:11,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 940/2500 [02:45<03:12,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 942/2500 [02:45<03:04,  8.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 943/2500 [02:46<04:38,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 944/2500 [02:46<05:41,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 946/2500 [02:47<04:39,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 950/2500 [02:47<03:58,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 952/2500 [02:47<03:36,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 953/2500 [02:48<04:10,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 955/2500 [02:48<05:27,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 957/2500 [02:48<04:01,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 963/2500 [02:49<03:27,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 965/2500 [02:50<04:35,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 966/2500 [02:50<06:02,  4.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 967/2500 [02:51<06:22,  4.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 970/2500 [02:51<04:18,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 975/2500 [02:52<04:14,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 978/2500 [02:52<04:22,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 981/2500 [02:53<02:58,  8.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 983/2500 [02:53<02:47,  9.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 985/2500 [02:53<03:42,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 987/2500 [02:54<03:45,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 988/2500 [02:54<04:04,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 989/2500 [02:54<05:49,  4.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 992/2500 [02:55<04:39,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 994/2500 [02:55<05:02,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 995/2500 [02:56<06:13,  4.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1001/2500 [02:56<02:53,  8.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1003/2500 [02:56<03:09,  7.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1006/2500 [02:56<02:36,  9.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1008/2500 [02:57<04:05,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1009/2500 [02:58<05:37,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1013/2500 [02:58<04:07,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1015/2500 [02:59<04:19,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1017/2500 [02:59<04:58,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1020/2500 [03:00<04:16,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1025/2500 [03:00<03:01,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1027/2500 [03:01<04:00,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1030/2500 [03:01<03:38,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1032/2500 [03:02<05:32,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████▏     | 1034/2500 [03:02<04:59,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1037/2500 [03:02<03:21,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1040/2500 [03:03<03:33,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1042/2500 [03:03<03:37,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1046/2500 [03:04<03:15,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1049/2500 [03:04<03:12,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1050/2500 [03:05<07:28,  3.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1056/2500 [03:06<03:15,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1059/2500 [03:06<03:11,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1064/2500 [03:07<02:50,  8.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1066/2500 [03:07<02:51,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1068/2500 [03:07<04:14,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1069/2500 [03:08<04:41,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1072/2500 [03:08<04:22,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1075/2500 [03:09<04:15,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1080/2500 [03:09<03:14,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1082/2500 [03:10<03:14,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1084/2500 [03:10<03:23,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1086/2500 [03:10<03:48,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1087/2500 [03:11<04:12,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▎     | 1092/2500 [03:11<02:55,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1094/2500 [03:11<03:12,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1097/2500 [03:12<03:32,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1098/2500 [03:12<04:00,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1100/2500 [03:13<04:19,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1104/2500 [03:13<02:47,  8.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1106/2500 [03:13<03:37,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1109/2500 [03:14<04:16,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1111/2500 [03:14<03:25,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1112/2500 [03:14<03:59,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1114/2500 [03:15<03:43,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1115/2500 [03:15<03:59,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1116/2500 [03:15<04:08,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1118/2500 [03:16<04:04,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1121/2500 [03:16<03:06,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1127/2500 [03:17<02:57,  7.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1128/2500 [03:17<04:08,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1130/2500 [03:17<03:56,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1132/2500 [03:18<04:22,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1135/2500 [03:19<04:17,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1138/2500 [03:19<03:29,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1139/2500 [03:19<03:58,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1140/2500 [03:19<04:32,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1143/2500 [03:20<03:35,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1144/2500 [03:20<04:08,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1146/2500 [03:20<04:14,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1148/2500 [03:21<03:01,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1150/2500 [03:21<03:04,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1151/2500 [03:21<03:24,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1154/2500 [03:22<03:46,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1156/2500 [03:22<04:47,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 1158/2500 [03:23<04:36,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 1161/2500 [03:23<03:40,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1162/2500 [03:23<04:01,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1164/2500 [03:24<04:35,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1171/2500 [03:24<02:21,  9.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1173/2500 [03:25<02:43,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1175/2500 [03:25<03:02,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1177/2500 [03:25<03:01,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1179/2500 [03:26<04:10,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1180/2500 [03:27<06:13,  3.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1184/2500 [03:27<03:36,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1186/2500 [03:27<03:16,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1188/2500 [03:27<02:34,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1190/2500 [03:28<02:51,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1193/2500 [03:28<02:19,  9.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1195/2500 [03:28<02:46,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1196/2500 [03:29<03:37,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1198/2500 [03:29<03:26,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1199/2500 [03:29<03:41,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1201/2500 [03:30<04:29,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1204/2500 [03:30<03:57,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1207/2500 [03:31<03:34,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1209/2500 [03:31<03:27,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▊     | 1213/2500 [03:31<02:58,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1215/2500 [03:32<02:38,  8.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▊     | 1218/2500 [03:32<02:27,  8.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1220/2500 [03:32<03:04,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1221/2500 [03:33<03:25,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1222/2500 [03:33<04:17,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1224/2500 [03:33<04:27,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1226/2500 [03:34<03:12,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1229/2500 [03:34<02:41,  7.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1231/2500 [03:34<03:05,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1233/2500 [03:35<04:05,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1234/2500 [03:35<05:00,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1237/2500 [03:36<04:54,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1241/2500 [03:36<03:32,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1246/2500 [03:37<02:38,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1249/2500 [03:37<02:51,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1250/2500 [03:38<03:54,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1253/2500 [03:38<03:33,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1254/2500 [03:39<04:26,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1255/2500 [03:39<04:23,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1256/2500 [03:39<04:29,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1259/2500 [03:39<03:28,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1261/2500 [03:40<03:08,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1264/2500 [03:40<03:25,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1265/2500 [03:40<03:33,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1269/2500 [03:41<02:29,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1273/2500 [03:42<03:48,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1274/2500 [03:42<03:37,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1276/2500 [03:42<03:52,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1277/2500 [03:43<04:16,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1278/2500 [03:43<04:27,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1279/2500 [03:43<04:37,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1280/2500 [03:43<05:00,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1287/2500 [03:44<01:56, 10.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1289/2500 [03:44<01:48, 11.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1291/2500 [03:44<02:28,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1293/2500 [03:46<04:57,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1294/2500 [03:46<05:38,  3.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1297/2500 [03:46<04:30,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1301/2500 [03:47<03:05,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1304/2500 [03:47<02:21,  8.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1306/2500 [03:47<02:06,  9.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1309/2500 [03:48<02:53,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1311/2500 [03:48<03:31,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1312/2500 [03:49<04:05,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1313/2500 [03:49<04:14,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1316/2500 [03:49<03:44,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1318/2500 [03:50<03:55,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1320/2500 [03:50<03:35,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1322/2500 [03:51<03:48,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1324/2500 [03:51<02:52,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1326/2500 [03:51<02:24,  8.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1327/2500 [03:51<02:45,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1331/2500 [03:52<02:21,  8.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1335/2500 [03:52<02:44,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▎    | 1338/2500 [03:53<03:22,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1340/2500 [03:54<04:01,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1342/2500 [03:54<03:29,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1343/2500 [03:54<04:22,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1344/2500 [03:55<04:42,  4.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1345/2500 [03:55<04:47,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1346/2500 [03:55<04:38,  4.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1348/2500 [03:55<03:48,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1351/2500 [03:56<03:02,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1354/2500 [03:56<03:30,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1357/2500 [03:57<02:43,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1359/2500 [03:57<02:35,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1361/2500 [03:57<02:52,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1362/2500 [03:58<03:45,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1364/2500 [03:58<03:46,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1367/2500 [03:58<02:33,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1371/2500 [03:59<02:32,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1373/2500 [03:59<03:16,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1375/2500 [04:00<03:12,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1376/2500 [04:00<03:32,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1380/2500 [04:00<02:13,  8.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1382/2500 [04:01<03:36,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1386/2500 [04:02<02:57,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1388/2500 [04:02<03:26,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1390/2500 [04:02<03:04,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1392/2500 [04:03<03:34,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1396/2500 [04:03<02:46,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1398/2500 [04:04<02:59,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1400/2500 [04:04<03:18,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1403/2500 [04:05<03:38,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1405/2500 [04:05<02:55,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1407/2500 [04:05<02:22,  7.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1409/2500 [04:05<02:01,  8.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1412/2500 [04:06<02:22,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1415/2500 [04:06<02:33,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1416/2500 [04:07<04:15,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1417/2500 [04:07<04:37,  3.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1419/2500 [04:07<03:37,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1420/2500 [04:08<04:04,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1422/2500 [04:08<04:28,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1425/2500 [04:09<03:29,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1428/2500 [04:09<02:21,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1433/2500 [04:09<01:40, 10.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1435/2500 [04:10<02:20,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1437/2500 [04:11<04:01,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1438/2500 [04:11<03:59,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1439/2500 [04:11<04:01,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1441/2500 [04:11<03:26,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1444/2500 [04:12<03:28,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1449/2500 [04:13<02:01,  8.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1452/2500 [04:13<01:30, 11.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1454/2500 [04:13<01:55,  9.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1456/2500 [04:14<03:14,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1458/2500 [04:14<03:16,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1459/2500 [04:15<03:43,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▊    | 1463/2500 [04:15<02:38,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1465/2500 [04:15<02:18,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1467/2500 [04:15<02:13,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1470/2500 [04:16<02:14,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1472/2500 [04:16<02:31,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1474/2500 [04:16<01:58,  8.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1476/2500 [04:17<03:09,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1478/2500 [04:18<03:41,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1484/2500 [04:18<01:39, 10.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1486/2500 [04:18<01:39, 10.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1488/2500 [04:18<01:44,  9.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1490/2500 [04:19<02:20,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1491/2500 [04:19<03:46,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1493/2500 [04:20<03:34,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1497/2500 [04:20<02:55,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1501/2500 [04:21<01:57,  8.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1503/2500 [04:21<01:38, 10.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1505/2500 [04:21<01:45,  9.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1508/2500 [04:22<02:13,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1509/2500 [04:22<03:03,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1510/2500 [04:22<03:27,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1512/2500 [04:23<04:27,  3.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1516/2500 [04:23<02:40,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1518/2500 [04:24<02:12,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1521/2500 [04:24<02:41,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1524/2500 [04:24<02:10,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1526/2500 [04:25<02:25,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1528/2500 [04:25<02:14,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1529/2500 [04:25<02:50,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 1532/2500 [04:26<02:39,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1533/2500 [04:26<03:11,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 1536/2500 [04:27<03:15,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1539/2500 [04:27<02:03,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1541/2500 [04:27<01:47,  8.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1543/2500 [04:28<02:05,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1546/2500 [04:28<01:57,  8.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1548/2500 [04:28<01:59,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1550/2500 [04:29<02:51,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1551/2500 [04:29<02:48,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1552/2500 [04:29<03:27,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1553/2500 [04:30<04:00,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1556/2500 [04:30<02:54,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1559/2500 [04:30<01:45,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1561/2500 [04:31<02:01,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1566/2500 [04:31<01:38,  9.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1569/2500 [04:32<02:59,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1573/2500 [04:32<02:13,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1574/2500 [04:33<02:15,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1575/2500 [04:33<02:40,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1579/2500 [04:33<02:21,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1581/2500 [04:34<02:28,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1583/2500 [04:34<02:19,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1584/2500 [04:34<02:57,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1588/2500 [04:35<02:11,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1591/2500 [04:35<02:44,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1592/2500 [04:36<02:43,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1594/2500 [04:36<02:25,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1596/2500 [04:36<02:44,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1599/2500 [04:37<02:03,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1603/2500 [04:37<01:59,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1604/2500 [04:37<02:02,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1605/2500 [04:38<02:15,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1606/2500 [04:38<03:39,  4.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1607/2500 [04:39<04:13,  3.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1610/2500 [04:39<03:34,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1612/2500 [04:40<03:11,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1614/2500 [04:40<02:09,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1616/2500 [04:40<01:45,  8.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1618/2500 [04:40<01:42,  8.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1622/2500 [04:41<02:00,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1624/2500 [04:41<02:53,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1627/2500 [04:42<03:34,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1630/2500 [04:42<02:14,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1632/2500 [04:43<02:33,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1634/2500 [04:43<02:54,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1637/2500 [04:44<02:20,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1640/2500 [04:44<02:01,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1643/2500 [04:44<02:03,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1644/2500 [04:45<02:02,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1645/2500 [04:45<03:11,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1647/2500 [04:46<03:11,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1652/2500 [04:46<02:02,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1654/2500 [04:46<01:41,  8.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1656/2500 [04:46<01:29,  9.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1662/2500 [04:47<01:29,  9.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1664/2500 [04:48<02:19,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1665/2500 [04:48<02:45,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1667/2500 [04:48<02:43,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1668/2500 [04:49<02:46,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1670/2500 [04:49<02:34,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1672/2500 [04:49<02:28,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1676/2500 [04:50<01:32,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1678/2500 [04:50<02:14,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1682/2500 [04:51<01:48,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1683/2500 [04:52<04:08,  3.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1687/2500 [04:52<02:19,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1691/2500 [04:52<01:41,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1693/2500 [04:53<02:01,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1694/2500 [04:53<02:24,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1697/2500 [04:54<02:03,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1698/2500 [04:54<02:12,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1700/2500 [04:54<02:35,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1701/2500 [04:54<02:30,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1704/2500 [04:55<01:57,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1706/2500 [04:55<02:21,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1709/2500 [04:56<01:46,  7.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 1710/2500 [04:56<01:50,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 1712/2500 [04:56<02:22,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 1715/2500 [04:57<01:59,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 1717/2500 [04:57<01:31,  8.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1719/2500 [04:57<01:27,  8.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1721/2500 [04:57<01:40,  7.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1722/2500 [04:58<02:13,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 1726/2500 [04:58<01:55,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 1729/2500 [04:59<02:15,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1730/2500 [04:59<02:35,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 1733/2500 [05:00<02:06,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 1739/2500 [05:00<01:18,  9.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 1741/2500 [05:01<02:16,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 1743/2500 [05:01<02:24,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 1745/2500 [05:02<02:46,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 1747/2500 [05:02<02:04,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 1750/2500 [05:03<01:53,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1752/2500 [05:03<01:57,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 1756/2500 [05:03<01:20,  9.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 1758/2500 [05:04<01:31,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 1760/2500 [05:04<02:26,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1764/2500 [05:05<01:43,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1766/2500 [05:06<02:27,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1769/2500 [05:06<01:41,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 1774/2500 [05:06<01:11, 10.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1776/2500 [05:07<01:55,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1778/2500 [05:07<01:46,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1779/2500 [05:07<02:14,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 1780/2500 [05:08<02:38,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 1784/2500 [05:08<01:52,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 1786/2500 [05:08<01:40,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1789/2500 [05:09<01:37,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1792/2500 [05:09<01:08, 10.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1794/2500 [05:09<01:04, 11.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1796/2500 [05:10<01:22,  8.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1798/2500 [05:10<01:57,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1802/2500 [05:11<01:42,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 1806/2500 [05:11<01:50,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 1808/2500 [05:12<02:07,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1815/2500 [05:12<01:02, 10.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1819/2500 [05:13<01:48,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1821/2500 [05:14<01:39,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1823/2500 [05:14<01:32,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1826/2500 [05:14<01:45,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1827/2500 [05:15<01:40,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1828/2500 [05:15<02:04,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1830/2500 [05:15<01:48,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1831/2500 [05:15<01:56,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1833/2500 [05:16<01:38,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 1835/2500 [05:16<01:52,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 1836/2500 [05:16<01:52,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 1839/2500 [05:17<01:34,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1844/2500 [05:17<01:04, 10.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1846/2500 [05:18<02:08,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1848/2500 [05:18<01:54,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1852/2500 [05:19<01:27,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1854/2500 [05:19<01:51,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1857/2500 [05:19<01:35,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 1859/2500 [05:20<02:17,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 1861/2500 [05:20<02:06,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 1865/2500 [05:21<01:25,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 1867/2500 [05:21<01:07,  9.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 1869/2500 [05:21<01:13,  8.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 1871/2500 [05:21<01:17,  8.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 1872/2500 [05:22<01:22,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 1875/2500 [05:22<01:34,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1876/2500 [05:23<02:21,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 1878/2500 [05:23<02:10,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1879/2500 [05:23<02:04,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 1882/2500 [05:24<01:45,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 1884/2500 [05:24<01:46,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 1885/2500 [05:24<01:38,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1888/2500 [05:25<01:47,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1890/2500 [05:25<01:25,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1895/2500 [05:26<01:11,  8.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1897/2500 [05:26<01:21,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1899/2500 [05:26<01:26,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1901/2500 [05:27<01:52,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 1903/2500 [05:27<02:01,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 1905/2500 [05:28<01:31,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▋  | 1908/2500 [05:28<01:27,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 1910/2500 [05:28<01:07,  8.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 1912/2500 [05:28<01:08,  8.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1914/2500 [05:29<01:02,  9.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1916/2500 [05:29<01:35,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1921/2500 [05:30<01:14,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1922/2500 [05:30<01:48,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1924/2500 [05:31<02:02,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1926/2500 [05:31<02:06,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1928/2500 [05:32<01:57,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 1932/2500 [05:32<01:16,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1935/2500 [05:32<00:58,  9.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 1937/2500 [05:32<01:00,  9.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1941/2500 [05:33<00:57,  9.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1943/2500 [05:34<01:55,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1945/2500 [05:34<01:47,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1946/2500 [05:34<02:07,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1950/2500 [05:35<01:23,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1952/2500 [05:35<01:17,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1955/2500 [05:35<01:15,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1957/2500 [05:36<01:25,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 1960/2500 [05:36<01:17,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 1961/2500 [05:36<01:19,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 1963/2500 [05:37<01:52,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 1965/2500 [05:37<01:31,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 1966/2500 [05:38<01:44,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1969/2500 [05:38<01:31,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1972/2500 [05:38<01:16,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1977/2500 [05:39<00:56,  9.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1978/2500 [05:39<01:15,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1979/2500 [05:39<01:33,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1982/2500 [05:40<01:24,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 1983/2500 [05:40<01:19,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 1986/2500 [05:41<01:36,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 1988/2500 [05:41<01:27,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 1990/2500 [05:41<01:13,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 1993/2500 [05:42<00:49, 10.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 1995/2500 [05:42<00:44, 11.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 1997/2500 [05:42<01:34,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2000/2500 [05:43<01:20,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2002/2500 [05:43<01:34,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2005/2500 [05:44<01:44,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2010/2500 [05:45<01:03,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2012/2500 [05:45<01:06,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2015/2500 [05:46<01:14,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2017/2500 [05:46<01:27,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2019/2500 [05:46<01:28,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2020/2500 [05:47<01:48,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2021/2500 [05:47<01:49,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2023/2500 [05:47<01:35,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2024/2500 [05:48<01:52,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2029/2500 [05:48<01:04,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2031/2500 [05:49<01:06,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2036/2500 [05:49<00:51,  8.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2040/2500 [05:50<01:16,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2042/2500 [05:51<01:35,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2046/2500 [05:51<01:14,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2049/2500 [05:52<01:06,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2050/2500 [05:52<01:13,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2051/2500 [05:52<01:27,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2053/2500 [05:52<01:14,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2054/2500 [05:53<01:20,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2057/2500 [05:53<01:15,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2058/2500 [05:53<01:17,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2061/2500 [05:54<01:03,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2064/2500 [05:54<00:49,  8.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2065/2500 [05:54<01:21,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2066/2500 [05:55<01:34,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2072/2500 [05:56<01:00,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2074/2500 [05:56<01:04,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2076/2500 [05:56<01:06,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2079/2500 [05:57<01:02,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2081/2500 [05:57<01:03,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2085/2500 [05:57<00:34, 11.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2087/2500 [05:58<01:08,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2090/2500 [05:59<01:21,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2093/2500 [05:59<01:14,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2094/2500 [05:59<01:30,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2096/2500 [06:00<01:25,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2099/2500 [06:00<00:57,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2103/2500 [06:00<00:31, 12.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2107/2500 [06:01<00:55,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2109/2500 [06:02<01:13,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2111/2500 [06:02<01:06,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2112/2500 [06:02<01:10,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2114/2500 [06:03<01:08,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2116/2500 [06:03<01:06,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2121/2500 [06:03<00:47,  7.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2123/2500 [06:04<01:08,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2126/2500 [06:04<00:54,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2128/2500 [06:05<01:08,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2130/2500 [06:05<01:01,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2132/2500 [06:06<01:10,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2133/2500 [06:06<01:12,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2134/2500 [06:06<01:21,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2137/2500 [06:06<00:54,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2138/2500 [06:07<00:58,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2140/2500 [06:07<01:12,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2142/2500 [06:07<00:58,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2147/2500 [06:08<00:30, 11.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2150/2500 [06:09<00:54,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2151/2500 [06:09<00:59,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2155/2500 [06:10<01:08,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▋ | 2159/2500 [06:10<00:53,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2160/2500 [06:10<00:52,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2165/2500 [06:11<00:40,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2167/2500 [06:11<00:33,  9.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2169/2500 [06:11<00:47,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2171/2500 [06:12<00:49,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2173/2500 [06:12<00:51,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2174/2500 [06:13<01:14,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2176/2500 [06:13<01:13,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2177/2500 [06:13<01:09,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2179/2500 [06:14<00:58,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2182/2500 [06:14<00:49,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2187/2500 [06:14<00:32,  9.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2189/2500 [06:15<00:30, 10.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2193/2500 [06:16<00:50,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2195/2500 [06:16<01:09,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2199/2500 [06:17<00:41,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2202/2500 [06:17<00:49,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2206/2500 [06:18<00:33,  8.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2210/2500 [06:18<00:32,  8.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2212/2500 [06:18<00:36,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2214/2500 [06:19<01:04,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2216/2500 [06:20<00:57,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 2218/2500 [06:20<00:52,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2219/2500 [06:20<00:48,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2220/2500 [06:20<01:00,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2223/2500 [06:21<00:51,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2228/2500 [06:21<00:34,  7.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2230/2500 [06:22<00:29,  9.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2232/2500 [06:22<00:37,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2234/2500 [06:23<00:46,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2236/2500 [06:23<00:43,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2238/2500 [06:23<00:52,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2240/2500 [06:24<00:56,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2246/2500 [06:25<00:33,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2250/2500 [06:25<00:29,  8.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2252/2500 [06:25<00:33,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2256/2500 [06:26<00:26,  9.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2258/2500 [06:27<00:52,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2259/2500 [06:27<01:03,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2263/2500 [06:28<00:51,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2264/2500 [06:28<00:59,  3.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2272/2500 [06:29<00:27,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2274/2500 [06:29<00:26,  8.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2276/2500 [06:29<00:32,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2277/2500 [06:30<00:43,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2278/2500 [06:30<00:45,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2282/2500 [06:31<00:35,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 2283/2500 [06:31<00:40,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 2284/2500 [06:31<00:49,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 2286/2500 [06:32<00:42,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2291/2500 [06:32<00:31,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2293/2500 [06:32<00:29,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2295/2500 [06:33<00:28,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2298/2500 [06:33<00:27,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2300/2500 [06:34<00:30,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2302/2500 [06:34<00:37,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2304/2500 [06:35<00:42,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2308/2500 [06:35<00:25,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2310/2500 [06:35<00:22,  8.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2313/2500 [06:36<00:28,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2315/2500 [06:36<00:26,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2319/2500 [06:36<00:21,  8.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2321/2500 [06:37<00:22,  8.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2322/2500 [06:37<00:38,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2325/2500 [06:38<00:31,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2330/2500 [06:39<00:34,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2334/2500 [06:39<00:21,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2336/2500 [06:39<00:24,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2338/2500 [06:40<00:22,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▎| 2341/2500 [06:40<00:26,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2343/2500 [06:41<00:25,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2346/2500 [06:41<00:19,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2348/2500 [06:41<00:25,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2350/2500 [06:42<00:34,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2351/2500 [06:42<00:33,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2353/2500 [06:43<00:27,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2355/2500 [06:43<00:27,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2357/2500 [06:43<00:27,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2360/2500 [06:44<00:19,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2364/2500 [06:44<00:13, 10.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2366/2500 [06:44<00:18,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2367/2500 [06:45<00:22,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2369/2500 [06:45<00:25,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2372/2500 [06:46<00:26,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2375/2500 [06:46<00:22,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2376/2500 [06:47<00:23,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2378/2500 [06:47<00:24,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2382/2500 [06:47<00:16,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2383/2500 [06:48<00:19,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2386/2500 [06:48<00:16,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2387/2500 [06:48<00:19,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2388/2500 [06:49<00:21,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2389/2500 [06:50<00:41,  2.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2393/2500 [06:50<00:22,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2395/2500 [06:50<00:19,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2398/2500 [06:51<00:16,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2401/2500 [06:51<00:11,  8.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2403/2500 [06:51<00:15,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2404/2500 [06:52<00:19,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2405/2500 [06:52<00:21,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2407/2500 [06:52<00:18,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2408/2500 [06:53<00:24,  3.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2413/2500 [06:54<00:16,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2417/2500 [06:54<00:12,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2419/2500 [06:54<00:11,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2421/2500 [06:55<00:16,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2423/2500 [06:55<00:12,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2425/2500 [06:56<00:11,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2426/2500 [06:56<00:18,  4.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2428/2500 [06:57<00:15,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2430/2500 [06:57<00:12,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2432/2500 [06:57<00:13,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2435/2500 [06:58<00:14,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2437/2500 [06:59<00:13,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2440/2500 [06:59<00:10,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2443/2500 [07:00<00:11,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2445/2500 [07:00<00:10,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2448/2500 [07:01<00:08,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2451/2500 [07:01<00:06,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2453/2500 [07:01<00:08,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2457/2500 [07:02<00:06,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2461/2500 [07:02<00:05,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2463/2500 [07:04<00:10,  3.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▊| 2466/2500 [07:04<00:06,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2468/2500 [07:04<00:06,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2471/2500 [07:05<00:05,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2474/2500 [07:05<00:04,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2476/2500 [07:06<00:03,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2479/2500 [07:06<00:03,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2480/2500 [07:06<00:02,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2482/2500 [07:07<00:03,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2483/2500 [07:07<00:04,  3.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2484/2500 [07:08<00:04,  3.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2487/2500 [07:08<00:02,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2489/2500 [07:08<00:02,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2490/2500 [07:09<00:02,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2491/2500 [07:09<00:02,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2495/2500 [07:10<00:00,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2496/2500 [07:10<00:00,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2499/2500 [07:10<00:00,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 2500/2500 [07:12<00:00,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "Evaluation metrics:  {'f1': 0.0, 'em': 0.0, 'acc': 0.2348}\n",
      "\u001b[32m2025-12-27 17:00:32.467\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.workflow.workflow_graph\u001b[0m:\u001b[36msave_module\u001b[0m:\u001b[36m1201\u001b[0m - \u001b[1mSaving SEWWorkFlowGraph to debug/optimized_sew_workflow_update_correct_round20_step20_gpt4omini_optzall.json\u001b[0m\n",
      "\u001b[32m2025-12-27 17:00:32.468\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.utils.utils\u001b[0m:\u001b[36mmake_parent_folder\u001b[0m:\u001b[36m19\u001b[0m - \u001b[1mcreating folder debug ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# obtain SEWOptimizer after having more roles\n",
    "optimizer = SEWOptimizer(\n",
    "    graph=sew_graph, \n",
    "    evaluator=evaluator, \n",
    "    llm=llm, \n",
    "    max_steps=20,\n",
    "    eval_rounds=1, \n",
    "    repr_scheme=\"python\", \n",
    "    optimize_mode=\"all\", \n",
    "    order=\"zero-order\",\n",
    "    max_rounds=20,\n",
    ")\n",
    "\n",
    "# with suppress_logger_info():\n",
    "#     metrics = optimizer.evaluate(dataset=humaneval, eval_mode=\"test\")\n",
    "# print(\"Evaluation metrics: \", metrics)\n",
    "\n",
    "# optimize the SEW workflow\n",
    "optimizer.optimize(dataset=benchmark)\n",
    "\n",
    "# evaluate the optimized SEW workflow\n",
    "with suppress_logger_info():\n",
    "    metrics = optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "print(\"Evaluation metrics: \", metrics)\n",
    "\n",
    "# save the optimized SEW workflow\n",
    "optimizer.save(\"debug/optimized_sew_workflow_update_correct_round20_step20_gpt4omini_optzall.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c5f272e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'f1': 0.0, 'em': 0.0, 'acc': 0.2348}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "463adce5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e9f6f8eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"/gpfs/radev/project/zhao/hs2234/evox/QA/gene_perturb/qa/norman_psa_single_qa.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "884db009",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"You are an expert in single-cell biology and functional genomics. In K562 cells, AHR is perturbed and the expression of ENSG00000134321 is measured. Does this perturbation cause a significant change in ENSG00000134321 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\""
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['question'].values[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5642cc11",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "answer\n",
       "Yes    2930\n",
       "No      220\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['answer'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2eeaf26f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import scanpy as sc\n",
    "adata = sc.read_h5ad(\"/home/tl688/pitl688/pert_test/data/norman/perturb_processed.h5ad\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6bb8ee40",
   "metadata": {},
   "outputs": [],
   "source": [
    "# adata.var\n",
    "map_dict = {}\n",
    "for i in adata.var.index:\n",
    "    gene = adata.var.loc[i]['gene_name']\n",
    "    map_dict[i] = gene"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "81675c54",
   "metadata": {},
   "outputs": [],
   "source": [
    "# more powerful model;\n",
    "# api/knowledge base query;\n",
    "# analyze system; "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "558d807a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ENSG00000239945': 'RP11-34P13.8',\n",
       " 'ENSG00000223764': 'RP11-54O7.3',\n",
       " 'ENSG00000187634': 'SAMD11',\n",
       " 'ENSG00000187642': 'PERM1',\n",
       " 'ENSG00000188290': 'HES4',\n",
       " 'ENSG00000187608': 'ISG15',\n",
       " 'ENSG00000273443': 'RP11-54O7.18',\n",
       " 'ENSG00000237330': 'RNF223',\n",
       " 'ENSG00000223823': 'LINC01342',\n",
       " 'ENSG00000205231': 'TTLL10-AS1',\n",
       " 'ENSG00000186891': 'TNFRSF18',\n",
       " 'ENSG00000186827': 'TNFRSF4',\n",
       " 'ENSG00000169962': 'TAS1R3',\n",
       " 'ENSG00000235098': 'ANKRD65',\n",
       " 'ENSG00000189409': 'MMP23B',\n",
       " 'ENSG00000269737': 'RP11-345P4.7',\n",
       " 'ENSG00000169885': 'CALML6',\n",
       " 'ENSG00000271806': 'RP5-892K4.1',\n",
       " 'ENSG00000234396': 'RP11-181G12.4',\n",
       " 'ENSG00000149527': 'PLCH2',\n",
       " 'ENSG00000272449': 'RP3-395M20.12',\n",
       " 'ENSG00000238164': 'RP3-395M20.8',\n",
       " 'ENSG00000157873': 'TNFRSF14',\n",
       " 'ENSG00000215912': 'TTC34',\n",
       " 'ENSG00000078900': 'TP73',\n",
       " 'ENSG00000235169': 'SMIM1',\n",
       " 'ENSG00000272153': 'RP1-286D6.5',\n",
       " 'ENSG00000116254': 'CHD5',\n",
       " 'ENSG00000225077': 'LINC00337',\n",
       " 'ENSG00000158292': 'GPR153',\n",
       " 'ENSG00000069812': 'HES2',\n",
       " 'ENSG00000187017': 'ESPN',\n",
       " 'ENSG00000173662': 'TAS1R1',\n",
       " 'ENSG00000270171': 'RP11-338N10.1',\n",
       " 'ENSG00000049249': 'TNFRSF9',\n",
       " 'ENSG00000232912': 'RP5-1115A15.1',\n",
       " 'ENSG00000228423': 'RP4-633I8.4',\n",
       " 'ENSG00000074800': 'ENO1',\n",
       " 'ENSG00000131686': 'CA6',\n",
       " 'ENSG00000180758': 'GPR157',\n",
       " 'ENSG00000228526': 'MIR34AHG',\n",
       " 'ENSG00000234546': 'RP3-510D11.2',\n",
       " 'ENSG00000049239': 'H6PD',\n",
       " 'ENSG00000179840': 'PIK3CD-AS1',\n",
       " 'ENSG00000280113': 'AL357140.1',\n",
       " 'ENSG00000142657': 'PGD',\n",
       " 'ENSG00000175262': 'C1orf127',\n",
       " 'ENSG00000277726': 'RP4-635E18.9',\n",
       " 'ENSG00000009724': 'MASP2',\n",
       " 'ENSG00000271895': 'RP4-635E18.8',\n",
       " 'ENSG00000116649': 'SRM',\n",
       " 'ENSG00000162490': 'DRAXIN',\n",
       " 'ENSG00000177000': 'MTHFR',\n",
       " 'ENSG00000162493': 'PDPN',\n",
       " 'ENSG00000175147': 'TMEM51-AS1',\n",
       " 'ENSG00000233485': 'RP3-467K16.2',\n",
       " 'ENSG00000228140': 'RP3-467K16.4',\n",
       " 'ENSG00000142634': 'EFHD2',\n",
       " 'ENSG00000142615': 'CELA2A',\n",
       " 'ENSG00000215704': 'CELA2B',\n",
       " 'ENSG00000271742': 'RP4-680D5.9',\n",
       " 'ENSG00000224621': 'RP11-276H7.3',\n",
       " 'ENSG00000261135': 'RP4-798A10.7',\n",
       " 'ENSG00000271732': 'RP5-1182A14.5',\n",
       " 'ENSG00000280226': 'AC004824.2',\n",
       " 'ENSG00000230424': 'RP1-43E13.2',\n",
       " 'ENSG00000188257': 'PLA2G2A',\n",
       " 'ENSG00000117215': 'PLA2G2D',\n",
       " 'ENSG00000187980': 'PLA2G2C',\n",
       " 'ENSG00000117242': 'PINK1-AS',\n",
       " 'ENSG00000236936': 'RP3-329E20.2',\n",
       " 'ENSG00000142789': 'CELA3A',\n",
       " 'ENSG00000070831': 'CDC42',\n",
       " 'ENSG00000162552': 'WNT4',\n",
       " 'ENSG00000173369': 'C1QB',\n",
       " 'ENSG00000249087': 'ZNF436-AS1',\n",
       " 'ENSG00000235052': 'RP1-150O5.3',\n",
       " 'ENSG00000117318': 'ID3',\n",
       " 'ENSG00000197880': 'MDS2',\n",
       " 'ENSG00000117305': 'HMGCL',\n",
       " 'ENSG00000179163': 'FUCA1',\n",
       " 'ENSG00000188822': 'CNR2',\n",
       " 'ENSG00000142677': 'IL22RA1',\n",
       " 'ENSG00000230023': 'RP11-10N16.2',\n",
       " 'ENSG00000020633': 'RUNX3',\n",
       " 'ENSG00000233755': 'RP4-799D16.1',\n",
       " 'ENSG00000231953': 'RP4-706G24.1',\n",
       " 'ENSG00000117614': 'SYF2',\n",
       " 'ENSG00000117616': 'RSRP1',\n",
       " 'ENSG00000233478': 'RP1-187B23.1',\n",
       " 'ENSG00000182749': 'PAQR7',\n",
       " 'ENSG00000117632': 'STMN1',\n",
       " 'ENSG00000158008': 'EXTL1',\n",
       " 'ENSG00000158022': 'TRIM63',\n",
       " 'ENSG00000142669': 'SH3BGRL3',\n",
       " 'ENSG00000169442': 'CD52',\n",
       " 'ENSG00000176083': 'ZNF683',\n",
       " 'ENSG00000198830': 'HMGN2',\n",
       " 'ENSG00000117713': 'ARID1A',\n",
       " 'ENSG00000226698': 'RP1-50O24.6',\n",
       " 'ENSG00000090273': 'NUDC',\n",
       " 'ENSG00000142765': 'SYTL1',\n",
       " 'ENSG00000000938': 'FGR',\n",
       " 'ENSG00000126709': 'IFI6',\n",
       " 'ENSG00000130775': 'THEMIS2',\n",
       " 'ENSG00000227050': 'RP11-460I13.2',\n",
       " 'ENSG00000169403': 'PTAFR',\n",
       " 'ENSG00000271398': 'RP5-1092A3.5',\n",
       " 'ENSG00000130766': 'SESN2',\n",
       " 'ENSG00000116329': 'OPRD1',\n",
       " 'ENSG00000233427': 'RP1-212P9.3',\n",
       " 'ENSG00000162511': 'LAPTM5',\n",
       " 'ENSG00000121769': 'FABP3',\n",
       " 'ENSG00000223907': 'LINC01226',\n",
       " 'ENSG00000235790': 'RP11-73M7.6',\n",
       " 'ENSG00000134668': 'SPOCD1',\n",
       " 'ENSG00000224066': 'RP4-622L5.7',\n",
       " 'ENSG00000222046': 'DCDC2B',\n",
       " 'ENSG00000183615': 'FAM167B',\n",
       " 'ENSG00000182866': 'LCK',\n",
       " 'ENSG00000175130': 'MARCKSL1',\n",
       " 'ENSG00000162526': 'TSSK3',\n",
       " 'ENSG00000160058': 'BSDC1',\n",
       " 'ENSG00000224409': 'RP11-114B7.6',\n",
       " 'ENSG00000121900': 'TMEM54',\n",
       " 'ENSG00000255811': 'RP1-34M23.5',\n",
       " 'ENSG00000271741': 'RP11-244H3.4',\n",
       " 'ENSG00000126067': 'PSMB2',\n",
       " 'ENSG00000092853': 'CLSPN',\n",
       " 'ENSG00000232335': 'RP11-435D7.3',\n",
       " 'ENSG00000116871': 'MAP7D1',\n",
       " 'ENSG00000214193': 'SH3D21',\n",
       " 'ENSG00000119535': 'CSF3R',\n",
       " 'ENSG00000233621': 'LINC01137',\n",
       " 'ENSG00000134690': 'CDCA8',\n",
       " 'ENSG00000158315': 'RHBDL2',\n",
       " 'ENSG00000127603': 'MACF1',\n",
       " 'ENSG00000226438': 'RP11-420K8.1',\n",
       " 'ENSG00000183682': 'BMP8A',\n",
       " 'ENSG00000168389': 'MFSD2A',\n",
       " 'ENSG00000131236': 'CAP1',\n",
       " 'ENSG00000049089': 'COL9A2',\n",
       " 'ENSG00000179862': 'CITED4',\n",
       " 'ENSG00000229528': 'RP5-1066H13.4',\n",
       " 'ENSG00000171790': 'SLFNL1',\n",
       " 'ENSG00000044012': 'GUCA2B',\n",
       " 'ENSG00000197273': 'GUCA2A',\n",
       " 'ENSG00000066185': 'ZMYND12',\n",
       " 'ENSG00000127125': 'PPCS',\n",
       " 'ENSG00000186409': 'CCDC30',\n",
       " 'ENSG00000065978': 'YBX1',\n",
       " 'ENSG00000164007': 'CLDN19',\n",
       " 'ENSG00000117394': 'SLC2A1',\n",
       " 'ENSG00000227533': 'SLC2A1-AS1',\n",
       " 'ENSG00000186973': 'FAM183A',\n",
       " 'ENSG00000243710': 'CFAP57',\n",
       " 'ENSG00000179178': 'TMEM125',\n",
       " 'ENSG00000253313': 'C1orf210',\n",
       " 'ENSG00000117399': 'CDC20',\n",
       " 'ENSG00000229444': 'RP11-184I16.4',\n",
       " 'ENSG00000117410': 'ATP6V0B',\n",
       " 'ENSG00000196517': 'SLC6A9',\n",
       " 'ENSG00000198520': 'C1orf228',\n",
       " 'ENSG00000142945': 'KIF2C',\n",
       " 'ENSG00000222009': 'BTBD19',\n",
       " 'ENSG00000070759': 'TESK2',\n",
       " 'ENSG00000117450': 'PRDX1',\n",
       " 'ENSG00000226957': 'RP4-533D7.4',\n",
       " 'ENSG00000117480': 'FAAH',\n",
       " 'ENSG00000269956': 'MKNK1-AS1',\n",
       " 'ENSG00000186118': 'TEX38',\n",
       " 'ENSG00000162366': 'PDZK1IP1',\n",
       " 'ENSG00000123473': 'STIL',\n",
       " 'ENSG00000123080': 'CDKN2C',\n",
       " 'ENSG00000203356': 'LINC01562',\n",
       " 'ENSG00000261664': 'TTC39A-AS1',\n",
       " 'ENSG00000169213': 'RAB3B',\n",
       " 'ENSG00000223390': 'RP11-91A18.4',\n",
       " 'ENSG00000228369': 'TXNDC12-AS1',\n",
       " 'ENSG00000116157': 'GPX7',\n",
       " 'ENSG00000182183': 'FAM159A',\n",
       " 'ENSG00000226754': 'RP5-1024G6.5',\n",
       " 'ENSG00000230138': 'RP11-117D22.2',\n",
       " 'ENSG00000116209': 'TMEM59',\n",
       " 'ENSG00000225632': 'RP5-997D24.3',\n",
       " 'ENSG00000143001': 'TMEM61',\n",
       " 'ENSG00000162399': 'BSND',\n",
       " 'ENSG00000162407': 'PLPP3',\n",
       " 'ENSG00000177606': 'JUN',\n",
       " 'ENSG00000232453': 'RP4-794H19.1',\n",
       " 'ENSG00000237352': 'LINC01358',\n",
       " 'ENSG00000132849': 'INADL',\n",
       " 'ENSG00000132855': 'ANGPTL3',\n",
       " 'ENSG00000228734': 'RP11-335E6.3',\n",
       " 'ENSG00000079739': 'PGM1',\n",
       " 'ENSG00000118473': 'SGIP1',\n",
       " 'ENSG00000203963': 'C1orf141',\n",
       " 'ENSG00000116717': 'GADD45A',\n",
       " 'ENSG00000237919': 'RP11-181B18.1',\n",
       " 'ENSG00000229051': 'RP5-952N6.1',\n",
       " 'ENSG00000235782': 'RP3-333A15.1',\n",
       " 'ENSG00000259030': 'FPGT-TNNI3K',\n",
       " 'ENSG00000137968': 'SLC44A5',\n",
       " 'ENSG00000178193': 'RP4-682C21.5',\n",
       " 'ENSG00000057468': 'MSH4',\n",
       " 'ENSG00000233099': 'RP11-375A5.1',\n",
       " 'ENSG00000235927': 'NEXN-AS1',\n",
       " 'ENSG00000137960': 'GIPC2',\n",
       " 'ENSG00000137959': 'IFI44L',\n",
       " 'ENSG00000137941': 'TTLL7',\n",
       " 'ENSG00000137976': 'DNASE2B',\n",
       " 'ENSG00000171502': 'COL24A1',\n",
       " 'ENSG00000016490': 'CLCA1',\n",
       " 'ENSG00000267272': 'LINC01140',\n",
       " 'ENSG00000143013': 'LMO4',\n",
       " 'ENSG00000117226': 'GBP3',\n",
       " 'ENSG00000117228': 'GBP1',\n",
       " 'ENSG00000162645': 'GBP2',\n",
       " 'ENSG00000231999': 'FLJ27354',\n",
       " 'ENSG00000189195': 'BTBD8',\n",
       " 'ENSG00000069712': 'KIAA1107',\n",
       " 'ENSG00000198691': 'ABCA4',\n",
       " 'ENSG00000117525': 'F3',\n",
       " 'ENSG00000235501': 'RP4-639F20.1',\n",
       " 'ENSG00000156869': 'FRRS1',\n",
       " 'ENSG00000122477': 'LRRC39',\n",
       " 'ENSG00000181656': 'GPR88',\n",
       " 'ENSG00000170989': 'S1PR1',\n",
       " 'ENSG00000260879': 'RP11-483I13.5',\n",
       " 'ENSG00000238122': 'RP11-483I13.2',\n",
       " 'ENSG00000143107': 'FNDC7',\n",
       " 'ENSG00000203897': 'SPATA42',\n",
       " 'ENSG00000273382': 'RP5-1065J22.8',\n",
       " 'ENSG00000270066': 'SCARNA2',\n",
       " 'ENSG00000179902': 'C1orf194',\n",
       " 'ENSG00000221986': 'MYBPHL',\n",
       " 'ENSG00000156097': 'GPR61',\n",
       " 'ENSG00000213366': 'GSTM2',\n",
       " 'ENSG00000134202': 'GSTM3',\n",
       " 'ENSG00000198758': 'EPS8L3',\n",
       " 'ENSG00000261055': 'RP11-195M16.3',\n",
       " 'ENSG00000143125': 'PROK1',\n",
       " 'ENSG00000143119': 'CD53',\n",
       " 'ENSG00000232811': 'RP11-96K19.2',\n",
       " 'ENSG00000064886': 'CHI3L2',\n",
       " 'ENSG00000143110': 'C1orf162',\n",
       " 'ENSG00000121933': 'TMIGD3',\n",
       " 'ENSG00000282608': 'ADORA3',\n",
       " 'ENSG00000197852': 'FAM212B',\n",
       " 'ENSG00000116489': 'CAPZA1',\n",
       " 'ENSG00000155366': 'RHOC',\n",
       " 'ENSG00000215866': 'LINC01356',\n",
       " 'ENSG00000134242': 'PTPN22',\n",
       " 'ENSG00000235527': 'HIPK1-AS1',\n",
       " 'ENSG00000116774': 'OLFML3',\n",
       " 'ENSG00000233154': 'RP4-655J12.4',\n",
       " 'ENSG00000224950': 'RP5-1086K13.1',\n",
       " 'ENSG00000116824': 'CD2',\n",
       " 'ENSG00000134256': 'CD101',\n",
       " 'ENSG00000236866': 'AL157902.3',\n",
       " 'ENSG00000183508': 'FAM46C',\n",
       " 'ENSG00000155761': 'SPAG17',\n",
       " 'ENSG00000092621': 'PHGDH',\n",
       " 'ENSG00000198019': 'FCGR1B',\n",
       " 'ENSG00000233029': 'RP11-439A17.9',\n",
       " 'ENSG00000230806': 'SRGAP2-AS1',\n",
       " 'ENSG00000263590': 'RP11-403I13.7',\n",
       " 'ENSG00000234571': 'RP5-998N21.4',\n",
       " 'ENSG00000277199': 'CH17-125A10.2',\n",
       " 'ENSG00000236140': 'RP11-89F3.2',\n",
       " 'ENSG00000174827': 'PDZK1',\n",
       " 'ENSG00000117281': 'CD160',\n",
       " 'ENSG00000198483': 'ANKRD35',\n",
       " 'ENSG00000121851': 'POLR3GL',\n",
       " 'ENSG00000265972': 'TXNIP',\n",
       " 'ENSG00000168509': 'HFE2',\n",
       " 'ENSG00000276509': 'CH17-353B19.1',\n",
       " 'ENSG00000131781': 'FMO5',\n",
       " 'ENSG00000227733': 'RP4-565E6.1',\n",
       " 'ENSG00000234283': 'RP11-495P10.6',\n",
       " 'ENSG00000231551': 'RP11-495P10.1',\n",
       " 'ENSG00000280649': 'AC245100.1',\n",
       " 'ENSG00000236334': 'PPIAL4G',\n",
       " 'ENSG00000178104': 'PDE4DIP',\n",
       " 'ENSG00000274265': 'CH17-189H20.1',\n",
       " 'ENSG00000150337': 'FCGR1A',\n",
       " 'ENSG00000233030': 'RP11-196G18.3',\n",
       " 'ENSG00000184260': 'HIST2H2AC',\n",
       " 'ENSG00000014914': 'MTMR11',\n",
       " 'ENSG00000023902': 'PLEKHO1',\n",
       " 'ENSG00000143401': 'ANP32E',\n",
       " 'ENSG00000276110': 'RP4-790G17.7',\n",
       " 'ENSG00000143369': 'ECM1',\n",
       " 'ENSG00000143382': 'ADAMTSL4',\n",
       " 'ENSG00000163131': 'CTSS',\n",
       " 'ENSG00000143387': 'CTSK',\n",
       " 'ENSG00000143409': 'FAM63A',\n",
       " 'ENSG00000163141': 'BNIPL',\n",
       " 'ENSG00000143443': 'C1orf56',\n",
       " 'ENSG00000197622': 'CDC42SE1',\n",
       " 'ENSG00000213190': 'MLLT11',\n",
       " 'ENSG00000143458': 'GABPB2',\n",
       " 'ENSG00000163155': 'LYSMD1',\n",
       " 'ENSG00000224645': 'RP11-126K1.8',\n",
       " 'ENSG00000250734': 'RP11-404E16.1',\n",
       " 'ENSG00000159409': 'CELF3',\n",
       " 'ENSG00000178796': 'RIIAD1',\n",
       " 'ENSG00000225556': 'C2CD4D',\n",
       " 'ENSG00000197747': 'S100A10',\n",
       " 'ENSG00000163191': 'S100A11',\n",
       " 'ENSG00000163216': 'SPRR2D',\n",
       " 'ENSG00000244094': 'SPRR2F',\n",
       " 'ENSG00000203783': 'PRR9',\n",
       " 'ENSG00000203782': 'LOR',\n",
       " 'ENSG00000163218': 'PGLYRP4',\n",
       " 'ENSG00000163220': 'S100A9',\n",
       " 'ENSG00000143546': 'S100A8',\n",
       " 'ENSG00000184330': 'S100A7A',\n",
       " 'ENSG00000143556': 'S100A7',\n",
       " 'ENSG00000197956': 'S100A6',\n",
       " 'ENSG00000196420': 'S100A5',\n",
       " 'ENSG00000196154': 'S100A4',\n",
       " 'ENSG00000188015': 'S100A3',\n",
       " 'ENSG00000189171': 'S100A13',\n",
       " 'ENSG00000160678': 'S100A1',\n",
       " 'ENSG00000279767': 'AL513523.2',\n",
       " 'ENSG00000282386': 'RP11-422P24.12',\n",
       " 'ENSG00000160712': 'IL6R',\n",
       " 'ENSG00000169291': 'SHE',\n",
       " 'ENSG00000143603': 'KCNN3',\n",
       " 'ENSG00000163346': 'PBXIP1',\n",
       " 'ENSG00000173207': 'CKS1B',\n",
       " 'ENSG00000163357': 'DCST1',\n",
       " 'ENSG00000169231': 'THBS3',\n",
       " 'ENSG00000236263': 'RP11-263K19.6',\n",
       " 'ENSG00000132718': 'SYT11',\n",
       " 'ENSG00000143622': 'RIT1',\n",
       " 'ENSG00000116584': 'ARHGEF2',\n",
       " 'ENSG00000132698': 'RAB25',\n",
       " 'ENSG00000160789': 'LMNA',\n",
       " 'ENSG00000183856': 'IQGAP3',\n",
       " 'ENSG00000143320': 'CRABP2',\n",
       " 'ENSG00000027869': 'SH2D2A',\n",
       " 'ENSG00000198400': 'NTRK1',\n",
       " 'ENSG00000027644': 'INSRR',\n",
       " 'ENSG00000160838': 'LRRC71',\n",
       " 'ENSG00000229961': 'RP11-71G12.1',\n",
       " 'ENSG00000132704': 'FCRL2',\n",
       " 'ENSG00000176320': 'RP11-404O13.5',\n",
       " 'ENSG00000163563': 'MNDA',\n",
       " 'ENSG00000163565': 'IFI16',\n",
       " 'ENSG00000213088': 'ACKR1',\n",
       " 'ENSG00000188004': 'C1orf204',\n",
       " 'ENSG00000243284': 'VSIG8',\n",
       " 'ENSG00000132681': 'ATP1A4',\n",
       " 'ENSG00000143318': 'CASQ1',\n",
       " 'ENSG00000258465': 'RP11-574F21.3',\n",
       " 'ENSG00000162739': 'SLAMF6',\n",
       " 'ENSG00000066294': 'CD84',\n",
       " 'ENSG00000117090': 'SLAMF1',\n",
       " 'ENSG00000228863': 'RP11-404F10.2',\n",
       " 'ENSG00000117091': 'CD48',\n",
       " 'ENSG00000122224': 'LY9',\n",
       " 'ENSG00000122223': 'CD244',\n",
       " 'ENSG00000179914': 'ITLN1',\n",
       " 'ENSG00000232188': 'RP11-312J18.6',\n",
       " 'ENSG00000158764': 'ITLN2',\n",
       " 'ENSG00000186517': 'ARHGAP30',\n",
       " 'ENSG00000143217': 'PVRL4',\n",
       " 'ENSG00000228917': 'RP11-544M22.8',\n",
       " 'ENSG00000162755': 'KLHDC9',\n",
       " 'ENSG00000158793': 'NIT1',\n",
       " 'ENSG00000158859': 'ADAMTS4',\n",
       " 'ENSG00000158869': 'FCER1G',\n",
       " 'ENSG00000188931': 'CFAP126',\n",
       " 'ENSG00000143226': 'FCGR2A',\n",
       " 'ENSG00000173110': 'HSPA6',\n",
       " 'ENSG00000203747': 'FCGR3A',\n",
       " 'ENSG00000072694': 'FCGR2B',\n",
       " 'ENSG00000132185': 'FCRLA',\n",
       " 'ENSG00000198929': 'NOS1AP',\n",
       " 'ENSG00000227094': 'RP11-565P22.2',\n",
       " 'ENSG00000185860': 'CCDC190',\n",
       " 'ENSG00000185630': 'PBX1',\n",
       " 'ENSG00000233693': 'RP11-506O24.1',\n",
       " 'ENSG00000234142': 'RP11-276E17.2',\n",
       " 'ENSG00000143194': 'MAEL',\n",
       " 'ENSG00000143162': 'CREG1',\n",
       " 'ENSG00000198771': 'RCSD1',\n",
       " 'ENSG00000197965': 'MPZL1',\n",
       " 'ENSG00000143158': 'MPC2',\n",
       " 'ENSG00000143184': 'XCL1',\n",
       " 'ENSG00000235736': 'RP1-10C16.1',\n",
       " 'ENSG00000225826': 'LINC00626',\n",
       " 'ENSG00000143153': 'ATP1B1',\n",
       " 'ENSG00000010932': 'FMO1',\n",
       " 'ENSG00000117586': 'TNFSF4',\n",
       " 'ENSG00000224977': 'RP11-160H22.3',\n",
       " 'ENSG00000152061': 'RABGAP1L',\n",
       " 'ENSG00000203737': 'GPR52',\n",
       " 'ENSG00000116161': 'CACYBP',\n",
       " 'ENSG00000236021': 'RP11-195C7.1',\n",
       " 'ENSG00000116194': 'ANGPTL1',\n",
       " 'ENSG00000186283': 'TOR3A',\n",
       " 'ENSG00000143322': 'ABL2',\n",
       " 'ENSG00000229407': 'RP11-12M5.3',\n",
       " 'ENSG00000169905': 'TOR1AIP2',\n",
       " 'ENSG00000116260': 'QSOX1',\n",
       " 'ENSG00000153029': 'MR1',\n",
       " 'ENSG00000162783': 'IER5',\n",
       " 'ENSG00000198216': 'CACNA1E',\n",
       " 'ENSG00000121446': 'RGSL1',\n",
       " 'ENSG00000135828': 'RNASEL',\n",
       " 'ENSG00000143333': 'RGS16',\n",
       " 'ENSG00000135824': 'RGS8',\n",
       " 'ENSG00000116701': 'NCF2',\n",
       " 'ENSG00000162704': 'ARPC5',\n",
       " 'ENSG00000143344': 'RGL1',\n",
       " 'ENSG00000198756': 'COLGALT2',\n",
       " 'ENSG00000135842': 'FAM129A',\n",
       " 'ENSG00000261024': 'GS1-279B7.1',\n",
       " 'ENSG00000116703': 'PDC',\n",
       " 'ENSG00000273198': 'GS1-304P7.3',\n",
       " 'ENSG00000150681': 'RGS18',\n",
       " 'ENSG00000232498': 'RP5-1011O1.2',\n",
       " 'ENSG00000090104': 'RGS1',\n",
       " 'ENSG00000116741': 'RGS2',\n",
       " 'ENSG00000234871': 'LINC01032',\n",
       " 'ENSG00000227240': 'RP11-563D10.1',\n",
       " 'ENSG00000227421': 'RP11-476B1.1',\n",
       " 'ENSG00000162687': 'KCNT2',\n",
       " 'ENSG00000000971': 'CFH',\n",
       " 'ENSG00000116785': 'CFHR3',\n",
       " 'ENSG00000134389': 'CFHR5',\n",
       " 'ENSG00000143278': 'F13B',\n",
       " 'ENSG00000066279': 'ASPM',\n",
       " 'ENSG00000134376': 'CRB1',\n",
       " 'ENSG00000081237': 'PTPRC',\n",
       " 'ENSG00000203721': 'LINC00862',\n",
       " 'ENSG00000118193': 'KIF14',\n",
       " 'ENSG00000081248': 'CACNA1S',\n",
       " 'ENSG00000232237': 'ASCL5',\n",
       " 'ENSG00000118194': 'TNNT2',\n",
       " 'ENSG00000159173': 'TNNI1',\n",
       " 'ENSG00000174307': 'PHLDA3',\n",
       " 'ENSG00000163431': 'LMOD1',\n",
       " 'ENSG00000234678': 'RP11-465N4.4',\n",
       " 'ENSG00000163435': 'ELF3',\n",
       " 'ENSG00000143851': 'PTPN7',\n",
       " 'ENSG00000117139': 'KDM5B',\n",
       " 'ENSG00000260021': 'RP11-480I12.10',\n",
       " 'ENSG00000143847': 'PPFIA4',\n",
       " 'ENSG00000122180': 'MYOG',\n",
       " 'ENSG00000133063': 'CHIT1',\n",
       " 'ENSG00000231507': 'LINC01353',\n",
       " 'ENSG00000159388': 'BTG2',\n",
       " 'ENSG00000122176': 'FMOD',\n",
       " 'ENSG00000058668': 'ATP2B4',\n",
       " 'ENSG00000122188': 'LAX1',\n",
       " 'ENSG00000257315': 'ZBED6',\n",
       " 'ENSG00000261065': 'RP11-74C13.4',\n",
       " 'ENSG00000143839': 'REN',\n",
       " 'ENSG00000170498': 'KISS1',\n",
       " 'ENSG00000174567': 'GOLT1A',\n",
       " 'ENSG00000226330': 'RP11-739N20.2',\n",
       " 'ENSG00000198625': 'MDM4',\n",
       " 'ENSG00000240710': 'RP11-430C7.4',\n",
       " 'ENSG00000240219': 'RP11-430C7.5',\n",
       " 'ENSG00000163531': 'NFASC',\n",
       " 'ENSG00000184144': 'CNTN2',\n",
       " 'ENSG00000186007': 'LEMD1',\n",
       " 'ENSG00000117266': 'CDK18',\n",
       " 'ENSG00000158715': 'SLC45A3',\n",
       " 'ENSG00000069275': 'NUCKS1',\n",
       " 'ENSG00000276600': 'RAB7B',\n",
       " 'ENSG00000196188': 'CTSE',\n",
       " 'ENSG00000263961': 'C1orf186',\n",
       " 'ENSG00000226780': 'RP11-38J22.3',\n",
       " 'ENSG00000261000': 'RP11-534L20.5',\n",
       " 'ENSG00000162889': 'MAPKAPK2',\n",
       " 'ENSG00000142224': 'IL19',\n",
       " 'ENSG00000162891': 'IL20',\n",
       " 'ENSG00000162892': 'IL24',\n",
       " 'ENSG00000162894': 'FCMR',\n",
       " 'ENSG00000196352': 'CD55',\n",
       " 'ENSG00000203710': 'CR1',\n",
       " 'ENSG00000236911': 'RP11-78B10.2',\n",
       " 'ENSG00000117335': 'CD46',\n",
       " 'ENSG00000203709': 'C1orf132',\n",
       " 'ENSG00000076356': 'PLXNA2',\n",
       " 'ENSG00000009790': 'TRAF3IP3',\n",
       " 'ENSG00000143473': 'KCNH1',\n",
       " 'ENSG00000153363': 'LINC00467',\n",
       " 'ENSG00000229258': 'RP11-552D8.1',\n",
       " 'ENSG00000226251': 'RP11-15I11.3',\n",
       " 'ENSG00000117691': 'NENF',\n",
       " 'ENSG00000162772': 'ATF3',\n",
       " 'ENSG00000237980': 'RP11-338C15.3',\n",
       " 'ENSG00000123685': 'BATF3',\n",
       " 'ENSG00000143494': 'VASH2',\n",
       " 'ENSG00000117707': 'PROX1',\n",
       " 'ENSG00000152104': 'PTPN14',\n",
       " 'ENSG00000117724': 'CENPF',\n",
       " 'ENSG00000228063': 'LYPLAL1-AS1',\n",
       " 'ENSG00000117791': 'MARC2',\n",
       " 'ENSG00000186205': 'MARC1',\n",
       " 'ENSG00000272823': 'RP11-295M18.6',\n",
       " 'ENSG00000238078': 'LINC01352',\n",
       " 'ENSG00000136630': 'HLX',\n",
       " 'ENSG00000234754': 'C1orf140',\n",
       " 'ENSG00000143498': 'TAF1A',\n",
       " 'ENSG00000197520': 'FAM177B',\n",
       " 'ENSG00000242861': 'RP11-285F7.2',\n",
       " 'ENSG00000243709': 'LEFTY1',\n",
       " 'ENSG00000143768': 'LEFTY2',\n",
       " 'ENSG00000185155': 'MIXL1',\n",
       " 'ENSG00000143772': 'ITPKB',\n",
       " 'ENSG00000228548': 'ITPKB-AS1',\n",
       " 'ENSG00000185888': 'PRSS38',\n",
       " 'ENSG00000143816': 'WNT9A',\n",
       " 'ENSG00000270104': 'RP11-245P10.6',\n",
       " 'ENSG00000237481': 'RP4-803J11.2',\n",
       " 'ENSG00000177614': 'PGBD5',\n",
       " 'ENSG00000135773': 'CAPN9',\n",
       " 'ENSG00000182118': 'FAM89A',\n",
       " 'ENSG00000119283': 'TRIM67',\n",
       " 'ENSG00000143674': 'KIAA1804',\n",
       " 'ENSG00000183780': 'SLC35F3',\n",
       " 'ENSG00000168264': 'IRF2BP2',\n",
       " 'ENSG00000228044': 'RP4-781K5.4',\n",
       " 'ENSG00000230628': 'RP4-781K5.6',\n",
       " 'ENSG00000237520': 'RP11-443B7.2',\n",
       " 'ENSG00000280587': 'LINC01348',\n",
       " 'ENSG00000054267': 'ARID4B',\n",
       " 'ENSG00000086619': 'ERO1B',\n",
       " 'ENSG00000215808': 'LINC01139',\n",
       " 'ENSG00000259776': 'RP11-544D21.2',\n",
       " 'ENSG00000117009': 'KMO',\n",
       " 'ENSG00000054282': 'SDCCAG8',\n",
       " 'ENSG00000117020': 'AKT3',\n",
       " 'ENSG00000179456': 'ZBTB18',\n",
       " 'ENSG00000185420': 'SMYD3',\n",
       " 'ENSG00000235096': 'RP11-83A16.1',\n",
       " 'ENSG00000235749': 'RP11-634B7.4',\n",
       " 'ENSG00000196071': 'OR2L13',\n",
       " 'ENSG00000203663': 'OR2L2',\n",
       " 'ENSG00000196944': 'OR2T4',\n",
       " 'ENSG00000233296': 'AC092159.2',\n",
       " 'ENSG00000272342': 'RP13-539J13.1',\n",
       " 'ENSG00000235078': 'AC142528.1',\n",
       " 'ENSG00000151360': 'ALLC',\n",
       " 'ENSG00000231532': 'LINC01249',\n",
       " 'ENSG00000205837': 'LINC00487',\n",
       " 'ENSG00000134321': 'RSAD2',\n",
       " 'ENSG00000231083': 'AC011747.6',\n",
       " 'ENSG00000235092': 'ID2-AS1',\n",
       " 'ENSG00000115738': 'ID2',\n",
       " 'ENSG00000143797': 'MBOAT2',\n",
       " 'ENSG00000151693': 'ASAP2',\n",
       " 'ENSG00000134330': 'IAH1',\n",
       " 'ENSG00000151694': 'ADAM17',\n",
       " 'ENSG00000240687': 'RP11-521D12.1',\n",
       " 'ENSG00000134317': 'GRHL1',\n",
       " 'ENSG00000171848': 'RRM2',\n",
       " 'ENSG00000115758': 'ODC1',\n",
       " 'ENSG00000243819': 'RN7SL832P',\n",
       " 'ENSG00000224184': 'MIR3681HG',\n",
       " 'ENSG00000223850': 'MYCNUT',\n",
       " 'ENSG00000197872': 'FAM49A',\n",
       " 'ENSG00000236204': 'LINC01376',\n",
       " 'ENSG00000219626': 'FAM228B',\n",
       " 'ENSG00000157856': 'DRC1',\n",
       " 'ENSG00000115155': 'OTOF',\n",
       " 'ENSG00000171303': 'KCNK3',\n",
       " 'ENSG00000115163': 'CENPA',\n",
       " 'ENSG00000084764': 'MAPRE3',\n",
       " 'ENSG00000231636': 'AGBL5-AS1',\n",
       " 'ENSG00000143994': 'ABHD1',\n",
       " 'ENSG00000163792': 'TCF23',\n",
       " 'ENSG00000163793': 'DNAJC5G',\n",
       " 'ENSG00000157992': 'KRTCAP3',\n",
       " 'ENSG00000075426': 'FOSL2',\n",
       " 'ENSG00000270210': 'RP11-373D23.3',\n",
       " 'ENSG00000163803': 'PLB1',\n",
       " 'ENSG00000213639': 'PPP1CB',\n",
       " 'ENSG00000163811': 'WDR43',\n",
       " 'ENSG00000233862': 'AC016907.3',\n",
       " 'ENSG00000119801': 'YPEL5',\n",
       " 'ENSG00000213626': 'LBH',\n",
       " 'ENSG00000158125': 'XDH',\n",
       " 'ENSG00000091106': 'NLRC4',\n",
       " 'ENSG00000152689': 'RASGRP3',\n",
       " 'ENSG00000237133': 'AC020594.5',\n",
       " 'ENSG00000228262': 'LINC01320',\n",
       " 'ENSG00000150938': 'CRIM1',\n",
       " 'ENSG00000115828': 'QPCT',\n",
       " 'ENSG00000163171': 'CDC42EP3',\n",
       " 'ENSG00000235848': 'RMDN2-AS1',\n",
       " 'ENSG00000138061': 'CYP1B1',\n",
       " 'ENSG00000232973': 'CYP1B1-AS1',\n",
       " 'ENSG00000214694': 'ARHGEF33',\n",
       " 'ENSG00000011566': 'MAP4K3',\n",
       " 'ENSG00000231312': 'AC007246.3',\n",
       " 'ENSG00000152154': 'TMEM178A',\n",
       " 'ENSG00000183023': 'SLC8A1',\n",
       " 'ENSG00000234362': 'AC104654.2',\n",
       " 'ENSG00000205086': 'C2orf91',\n",
       " 'ENSG00000162878': 'PKDCC',\n",
       " 'ENSG00000162881': 'OXER1',\n",
       " 'ENSG00000162882': 'HAAO',\n",
       " 'ENSG00000231826': 'AC016735.2',\n",
       " 'ENSG00000230587': 'AC093609.1',\n",
       " 'ENSG00000152518': 'ZFP36L2',\n",
       " 'ENSG00000152527': 'PLEKHH2',\n",
       " 'ENSG00000138079': 'SLC3A1',\n",
       " 'ENSG00000236502': 'SIX3-AS1',\n",
       " 'ENSG00000225156': 'AC012354.6',\n",
       " 'ENSG00000116016': 'EPAS1',\n",
       " 'ENSG00000187600': 'TMEM247',\n",
       " 'ENSG00000260977': 'RP11-333I13.1',\n",
       " 'ENSG00000222005': 'LINC01118',\n",
       " 'ENSG00000273269': 'RP11-761B3.1',\n",
       " 'ENSG00000143933': 'CALM2',\n",
       " 'ENSG00000236824': 'BCYRN1',\n",
       " 'ENSG00000230773': 'AC079807.4',\n",
       " 'ENSG00000282828': 'RP11-443F4.1',\n",
       " 'ENSG00000272156': 'RP11-477N3.1',\n",
       " 'ENSG00000234943': 'AC092839.4',\n",
       " 'ENSG00000214595': 'EML6',\n",
       " 'ENSG00000115310': 'RTN4',\n",
       " 'ENSG00000272180': 'RP11-481J13.1',\n",
       " 'ENSG00000222030': 'AC007131.1',\n",
       " 'ENSG00000270820': 'RP11-355B11.2',\n",
       " 'ENSG00000226622': 'AC092155.4',\n",
       " 'ENSG00000231609': 'AC009501.4',\n",
       " 'ENSG00000169764': 'UGP2',\n",
       " 'ENSG00000225889': 'AC074289.1',\n",
       " 'ENSG00000119862': 'LGALSL',\n",
       " 'ENSG00000260101': 'RP11-568N6.1',\n",
       " 'ENSG00000234572': 'AC007880.1',\n",
       " 'ENSG00000143995': 'MEIS1',\n",
       " 'ENSG00000230749': 'MEIS1-AS2',\n",
       " 'ENSG00000232046': 'AC007392.3',\n",
       " 'ENSG00000236780': 'AC078941.1',\n",
       " 'ENSG00000115956': 'PLEK',\n",
       " 'ENSG00000163219': 'ARHGAP25',\n",
       " 'ENSG00000163217': 'BMP10',\n",
       " 'ENSG00000183607': 'GKN2',\n",
       " 'ENSG00000115977': 'AAK1',\n",
       " 'ENSG00000231024': 'AC092431.3',\n",
       " 'ENSG00000059728': 'MXD1',\n",
       " 'ENSG00000116039': 'ATP6V1B1',\n",
       " 'ENSG00000124357': 'NAGK',\n",
       " 'ENSG00000272735': 'RP11-467P9.1',\n",
       " 'ENSG00000187833': 'C2orf78',\n",
       " 'ENSG00000235499': 'AC073046.25',\n",
       " 'ENSG00000144045': 'DQX1',\n",
       " 'ENSG00000159374': 'M1AP',\n",
       " 'ENSG00000159399': 'HK2',\n",
       " 'ENSG00000270571': 'RP11-355F16.1',\n",
       " 'ENSG00000115363': 'EVA1A',\n",
       " 'ENSG00000176204': 'LRRTM4',\n",
       " 'ENSG00000066032': 'CTNNA2',\n",
       " 'ENSG00000034510': 'TMSB10',\n",
       " 'ENSG00000042493': 'CAPG',\n",
       " 'ENSG00000118640': 'VAMP8',\n",
       " 'ENSG00000168899': 'VAMP5',\n",
       " 'ENSG00000115525': 'ST3GAL5',\n",
       " 'ENSG00000232504': 'ST3GAL5-AS1',\n",
       " 'ENSG00000272564': 'RP11-548P2.2',\n",
       " 'ENSG00000068615': 'REEP1',\n",
       " 'ENSG00000273445': 'RP11-1399P15.1',\n",
       " 'ENSG00000172086': 'KRCC1',\n",
       " 'ENSG00000211592': 'IGKC',\n",
       " 'ENSG00000233757': 'AC092835.2',\n",
       " 'ENSG00000274286': 'ADRA2B',\n",
       " 'ENSG00000158158': 'CNNM4',\n",
       " 'ENSG00000273265': 'RP11-353K11.1',\n",
       " 'ENSG00000168754': 'FAM178B',\n",
       " 'ENSG00000168658': 'VWA3B',\n",
       " 'ENSG00000196872': 'KIAA1211L',\n",
       " 'ENSG00000185674': 'LYG2',\n",
       " 'ENSG00000230393': 'AC092667.2',\n",
       " 'ENSG00000196460': 'RFX8',\n",
       " 'ENSG00000281162': 'LINC01127',\n",
       " 'ENSG00000115590': 'IL1R2',\n",
       " 'ENSG00000224850': 'AC108051.3',\n",
       " 'ENSG00000224509': 'AC010884.1',\n",
       " 'ENSG00000233339': 'AC018878.3',\n",
       " 'ENSG00000232001': 'AC108868.6',\n",
       " 'ENSG00000144057': 'ST6GAL2',\n",
       " 'ENSG00000198203': 'SULT1C2',\n",
       " 'ENSG00000198075': 'SULT1C4',\n",
       " 'ENSG00000169756': 'LIMS1',\n",
       " 'ENSG00000282033': 'RP11-506F3.1',\n",
       " 'ENSG00000015568': 'RGPD5',\n",
       " 'ENSG00000256977': 'LIMS3',\n",
       " 'ENSG00000144061': 'NPHP1',\n",
       " 'ENSG00000261760': 'RP11-1223D19.1',\n",
       " 'ENSG00000153094': 'BCL2L11',\n",
       " 'ENSG00000225744': 'AC017002.4',\n",
       " 'ENSG00000188177': 'ZC3H6',\n",
       " 'ENSG00000169629': 'RGPD8',\n",
       " 'ENSG00000228251': 'AC012442.6',\n",
       " 'ENSG00000144136': 'SLC20A1',\n",
       " 'ENSG00000125538': 'IL1B',\n",
       " 'ENSG00000136689': 'IL1RN',\n",
       " 'ENSG00000234997': 'AC016745.3',\n",
       " 'ENSG00000272563': 'RP11-480C16.1',\n",
       " 'ENSG00000227359': 'AC017074.2',\n",
       " 'ENSG00000244063': 'AC024704.2',\n",
       " 'ENSG00000228857': 'AC104653.1',\n",
       " 'ENSG00000259094': 'RP11-77A13.1',\n",
       " 'ENSG00000229867': 'STEAP3-AS1',\n",
       " 'ENSG00000155368': 'DBI',\n",
       " 'ENSG00000080293': 'SCTR',\n",
       " 'ENSG00000074047': 'GLI2',\n",
       " 'ENSG00000265451': 'RP11-204L24.2',\n",
       " 'ENSG00000224655': 'AC018737.3',\n",
       " 'ENSG00000136732': 'GYPC',\n",
       " 'ENSG00000186684': 'CYP27C1',\n",
       " 'ENSG00000115718': 'PROC',\n",
       " 'ENSG00000272789': 'RP11-286H15.1',\n",
       " 'ENSG00000152076': 'CCDC74B',\n",
       " 'ENSG00000178171': 'AMER3',\n",
       " 'ENSG00000183840': 'GPR39',\n",
       " 'ENSG00000226806': 'AC011893.3',\n",
       " 'ENSG00000121966': 'CXCR4',\n",
       " 'ENSG00000144229': 'THSD7B',\n",
       " 'ENSG00000150540': 'HNMT',\n",
       " 'ENSG00000115919': 'KYNU',\n",
       " 'ENSG00000169554': 'ZEB2',\n",
       " 'ENSG00000226674': 'TEX41',\n",
       " 'ENSG00000168280': 'KIF5C',\n",
       " 'ENSG00000231969': 'AC144449.1',\n",
       " 'ENSG00000123610': 'TNFAIP6',\n",
       " 'ENSG00000183091': 'NEB',\n",
       " 'ENSG00000162989': 'KCNJ3',\n",
       " 'ENSG00000226383': 'AC093375.1',\n",
       " 'ENSG00000153234': 'NR4A2',\n",
       " 'ENSG00000136541': 'ERMN',\n",
       " 'ENSG00000115165': 'CYTIP',\n",
       " 'ENSG00000271320': 'RP11-1152H14.1',\n",
       " 'ENSG00000153237': 'CCDC148',\n",
       " 'ENSG00000123636': 'BAZ2B',\n",
       " 'ENSG00000224152': 'AC009506.1',\n",
       " 'ENSG00000115271': 'GCA',\n",
       " 'ENSG00000184611': 'KCNH7',\n",
       " 'ENSG00000115339': 'GALNT3',\n",
       " 'ENSG00000169432': 'SCN9A',\n",
       " 'ENSG00000235335': 'AC016723.4',\n",
       " 'ENSG00000152253': 'SPC25',\n",
       " 'ENSG00000073734': 'ABCB11',\n",
       " 'ENSG00000073737': 'DHRS9',\n",
       " 'ENSG00000235321': 'AC007556.3',\n",
       " 'ENSG00000154479': 'CCDC173',\n",
       " 'ENSG00000231898': 'AC012594.1',\n",
       " 'ENSG00000213981': 'AC007277.3',\n",
       " 'ENSG00000222033': 'LINC01124',\n",
       " 'ENSG00000128683': 'GAD1',\n",
       " 'ENSG00000071967': 'CYBRD1',\n",
       " 'ENSG00000115844': 'DLX2',\n",
       " 'ENSG00000232788': 'AC078883.3',\n",
       " 'ENSG00000224638': 'AC106900.6',\n",
       " 'ENSG00000271151': 'RP11-394I13.2',\n",
       " 'ENSG00000231453': 'LINC01305',\n",
       " 'ENSG00000115935': 'WIPF1',\n",
       " 'ENSG00000128645': 'HOXD1',\n",
       " 'ENSG00000116044': 'NFE2L2',\n",
       " 'ENSG00000237655': 'AC073834.3',\n",
       " 'ENSG00000128655': 'PDE11A',\n",
       " 'ENSG00000229941': 'AC012499.1',\n",
       " 'ENSG00000271011': 'RP11-171I2.5',\n",
       " 'ENSG00000163492': 'CCDC141',\n",
       " 'ENSG00000115232': 'ITGA4',\n",
       " 'ENSG00000162992': 'NEUROD1',\n",
       " 'ENSG00000162998': 'FRZB',\n",
       " 'ENSG00000188738': 'FSIP2',\n",
       " 'ENSG00000231646': 'FSIP2-AS1',\n",
       " 'ENSG00000237877': 'LINC01473',\n",
       " 'ENSG00000144369': 'FAM171B',\n",
       " 'ENSG00000003436': 'TFPI',\n",
       " 'ENSG00000138449': 'SLC40A1',\n",
       " 'ENSG00000235852': 'AC005540.3',\n",
       " 'ENSG00000138378': 'STAT4',\n",
       " 'ENSG00000128641': 'MYO1B',\n",
       " 'ENSG00000173559': 'NABP1',\n",
       " 'ENSG00000235056': 'AC010983.1',\n",
       " 'ENSG00000223466': 'AC064834.2',\n",
       " 'ENSG00000272211': 'RP11-347P5.1',\n",
       " 'ENSG00000138411': 'HECW2',\n",
       " 'ENSG00000229056': 'AC020571.3',\n",
       " 'ENSG00000187944': 'C2orf66',\n",
       " 'ENSG00000065413': 'ANKRD44',\n",
       " 'ENSG00000225979': 'AC010746.3',\n",
       " 'ENSG00000144381': 'HSPD1',\n",
       " 'ENSG00000270757': 'HSPE1-MOB4',\n",
       " 'ENSG00000115896': 'PLCL1',\n",
       " 'ENSG00000163535': 'SGOL2',\n",
       " 'ENSG00000232719': 'RP11-13J8.1',\n",
       " 'ENSG00000003402': 'CFLAR',\n",
       " 'ENSG00000226312': 'CFLAR-AS1',\n",
       " 'ENSG00000155749': 'ALS2CR12',\n",
       " 'ENSG00000155754': 'ALS2CR11',\n",
       " 'ENSG00000138395': 'CDK15',\n",
       " 'ENSG00000273209': 'RP11-107N15.1',\n",
       " 'ENSG00000155760': 'FZD7',\n",
       " 'ENSG00000182329': 'KIAA2012',\n",
       " 'ENSG00000222035': 'AC079354.3',\n",
       " 'ENSG00000272966': 'RP11-686O6.1',\n",
       " 'ENSG00000138380': 'CARF',\n",
       " 'ENSG00000178562': 'CD28',\n",
       " 'ENSG00000204186': 'ZDBF2',\n",
       " 'ENSG00000260171': 'RP11-95H11.1',\n",
       " 'ENSG00000144410': 'CPO',\n",
       " 'ENSG00000118263': 'KLF7',\n",
       " 'ENSG00000234902': 'AC007879.2',\n",
       " 'ENSG00000224137': 'AC079767.4',\n",
       " 'ENSG00000144407': 'PTH2R',\n",
       " 'ENSG00000144406': 'UNC80',\n",
       " 'ENSG00000229127': 'AC007038.7',\n",
       " 'ENSG00000030419': 'IKZF2',\n",
       " 'ENSG00000144451': 'SPAG16',\n",
       " 'ENSG00000115414': 'FN1',\n",
       " 'ENSG00000163449': 'TMEM169',\n",
       " 'ENSG00000241520': 'AC098820.4',\n",
       " 'ENSG00000115457': 'IGFBP2',\n",
       " 'ENSG00000115461': 'IGFBP5',\n",
       " 'ENSG00000118245': 'TNP1',\n",
       " 'ENSG00000259855': 'RP11-574O16.1',\n",
       " 'ENSG00000079308': 'TNS1',\n",
       " 'ENSG00000180871': 'CXCR2',\n",
       " 'ENSG00000179921': 'GPBAR1',\n",
       " 'ENSG00000127831': 'VIL1',\n",
       " 'ENSG00000144580': 'RQCD1',\n",
       " 'ENSG00000115596': 'WNT6',\n",
       " 'ENSG00000135925': 'WNT10A',\n",
       " 'ENSG00000171450': 'CDK5R2',\n",
       " 'ENSG00000163497': 'FEV',\n",
       " 'ENSG00000163499': 'CRYBA2',\n",
       " 'ENSG00000135924': 'DNAJB2',\n",
       " 'ENSG00000230432': 'AC114803.3',\n",
       " 'ENSG00000144591': 'GMPPA',\n",
       " 'ENSG00000072182': 'ASIC4',\n",
       " 'ENSG00000114923': 'SLC4A3',\n",
       " 'ENSG00000163082': 'SGPP2',\n",
       " 'ENSG00000152049': 'KCNE4',\n",
       " 'ENSG00000144460': 'NYAP2',\n",
       " 'ENSG00000232023': 'AC009410.1',\n",
       " 'ENSG00000226125': 'AC098823.3',\n",
       " 'ENSG00000232520': 'AC012507.3',\n",
       " 'ENSG00000115053': 'NCL',\n",
       " 'ENSG00000187514': 'PTMA',\n",
       " 'ENSG00000196811': 'CHRNG',\n",
       " 'ENSG00000115474': 'KCNJ13',\n",
       " 'ENSG00000072080': 'SPP2',\n",
       " 'ENSG00000232893': 'IQCA1-AS1',\n",
       " 'ENSG00000124835': 'AC105760.3',\n",
       " 'ENSG00000163359': 'COL6A3',\n",
       " 'ENSG00000234949': 'AC104667.3',\n",
       " 'ENSG00000132329': 'RAMP1',\n",
       " 'ENSG00000168427': 'KLHL30',\n",
       " 'ENSG00000280260': 'AC017028.10',\n",
       " 'ENSG00000063660': 'GPC1',\n",
       " 'ENSG00000178623': 'GPR35',\n",
       " 'ENSG00000185176': 'AQP12B',\n",
       " 'ENSG00000219159': 'AC011298.2',\n",
       " 'ENSG00000172482': 'AGXT',\n",
       " 'ENSG00000091181': 'IL5RA',\n",
       " 'ENSG00000223727': 'AC026188.1',\n",
       " 'ENSG00000231249': 'ITPR1-AS1',\n",
       " 'ENSG00000150995': 'ITPR1',\n",
       " 'ENSG00000235947': 'EGOT',\n",
       " 'ENSG00000235978': 'AC018816.3',\n",
       " 'ENSG00000134107': 'BHLHE40',\n",
       " 'ENSG00000233912': 'AC026202.3',\n",
       " 'ENSG00000268509': 'RP11-572B2.1',\n",
       " 'ENSG00000196277': 'GRM7',\n",
       " 'ENSG00000071282': 'LMCD1',\n",
       " 'ENSG00000196220': 'SRGAP3',\n",
       " 'ENSG00000206573': 'THUMPD3-AS1',\n",
       " 'ENSG00000269886': 'RP11-266J6.2',\n",
       " 'ENSG00000163703': 'CRELD1',\n",
       " 'ENSG00000206567': 'AC022007.5',\n",
       " 'ENSG00000163705': 'FANCD2OS',\n",
       " 'ENSG00000157017': 'GHRL',\n",
       " 'ENSG00000132164': 'SLC6A11',\n",
       " 'ENSG00000157150': 'TIMP4',\n",
       " 'ENSG00000132170': 'PPARG',\n",
       " 'ENSG00000163520': 'FBLN2',\n",
       " 'ENSG00000224514': 'LINC00620',\n",
       " 'ENSG00000251576': 'LINC01267',\n",
       " 'ENSG00000144596': 'GRIP2',\n",
       " 'ENSG00000154783': 'FGD5',\n",
       " 'ENSG00000249786': 'EAF1-AS1',\n",
       " 'ENSG00000206561': 'COLQ',\n",
       " 'ENSG00000206560': 'ANKRD28',\n",
       " 'ENSG00000233153': 'UBE2E2-AS1',\n",
       " 'ENSG00000224074': 'LINC00691',\n",
       " 'ENSG00000077092': 'RARB',\n",
       " 'ENSG00000163491': 'NEK10',\n",
       " 'ENSG00000225548': 'AC098973.2',\n",
       " 'ENSG00000144642': 'RBMS3',\n",
       " 'ENSG00000163513': 'TGFBR2',\n",
       " 'ENSG00000144645': 'OSBPL10',\n",
       " 'ENSG00000091317': 'CMTM6',\n",
       " 'ENSG00000183813': 'CCR4',\n",
       " 'ENSG00000236452': 'AC123023.1',\n",
       " 'ENSG00000172995': 'ARPP21',\n",
       " 'ENSG00000163673': 'DCLK3',\n",
       " 'ENSG00000279727': 'RP11-640L9.1',\n",
       " 'ENSG00000168016': 'TRANK1',\n",
       " 'ENSG00000271993': 'RP11-285J16.1',\n",
       " 'ENSG00000271653': 'RP11-259K5.1',\n",
       " 'ENSG00000060971': 'ACAA1',\n",
       " 'ENSG00000172940': 'SLC22A13',\n",
       " 'ENSG00000168356': 'SCN11A',\n",
       " 'ENSG00000144655': 'CSRNP1',\n",
       " 'ENSG00000168334': 'XIRP1',\n",
       " 'ENSG00000168314': 'MOBP',\n",
       " 'ENSG00000114784': 'EIF1B',\n",
       " 'ENSG00000177842': 'ZNF620',\n",
       " 'ENSG00000168036': 'CTNNB1',\n",
       " 'ENSG00000157093': 'LYZL4',\n",
       " 'ENSG00000280571': 'RP4-613B23.5',\n",
       " 'ENSG00000225611': 'RP11-70C1.1',\n",
       " 'ENSG00000234617': 'SNRK-AS1',\n",
       " 'ENSG00000225873': 'LINC00694',\n",
       " 'ENSG00000163815': 'CLEC3B',\n",
       " 'ENSG00000163814': 'CDCP1',\n",
       " 'ENSG00000249992': 'TMEM158',\n",
       " 'ENSG00000163817': 'SLC6A20',\n",
       " 'ENSG00000173585': 'CCR9',\n",
       " 'ENSG00000163823': 'CCR1',\n",
       " 'ENSG00000223552': 'RP11-24F11.2',\n",
       " 'ENSG00000121797': 'CCRL2',\n",
       " 'ENSG00000178038': 'ALS2CL',\n",
       " 'ENSG00000160801': 'PTH1R',\n",
       " 'ENSG00000260236': 'RP11-708J19.1',\n",
       " 'ENSG00000164047': 'CAMP',\n",
       " 'ENSG00000145040': 'UCN2',\n",
       " 'ENSG00000221883': 'ARIH2OS',\n",
       " 'ENSG00000235236': 'RP13-131K19.1',\n",
       " 'ENSG00000272434': 'RP13-131K19.6',\n",
       " 'ENSG00000226913': 'BSN-AS2',\n",
       " 'ENSG00000187492': 'CDHR4',\n",
       " 'ENSG00000182179': 'UBA7',\n",
       " 'ENSG00000228008': 'CTD-2330K9.3',\n",
       " 'ENSG00000281691': 'RBM5-AS1',\n",
       " 'ENSG00000235016': 'SEMA3F-AS1',\n",
       " 'ENSG00000001617': 'SEMA3F',\n",
       " 'ENSG00000114353': 'GNAI2',\n",
       " 'ENSG00000230454': 'U73166.2',\n",
       " 'ENSG00000012171': 'SEMA3B',\n",
       " 'ENSG00000186792': 'HYAL3',\n",
       " 'ENSG00000114735': 'HEMK1',\n",
       " 'ENSG00000114737': 'CISH',\n",
       " 'ENSG00000145050': 'MANF',\n",
       " 'ENSG00000184345': 'IQCF2',\n",
       " 'ENSG00000023330': 'ALAS1',\n",
       " 'ENSG00000055955': 'ITIH4',\n",
       " 'ENSG00000272305': 'RP11-894J14.5',\n",
       " 'ENSG00000163931': 'TKT',\n",
       " 'ENSG00000157388': 'CACNA1D',\n",
       " 'ENSG00000186451': 'SPATA12',\n",
       " 'ENSG00000163666': 'HESX1',\n",
       " 'ENSG00000163687': 'DNASE1L3',\n",
       " 'ENSG00000273493': 'RP11-80H18.4',\n",
       " 'ENSG00000168297': 'PXK',\n",
       " 'ENSG00000272182': 'RP11-802O23.3',\n",
       " 'ENSG00000168309': 'FAM107A',\n",
       " 'ENSG00000163632': 'C3orf49',\n",
       " 'ENSG00000243410': 'RP11-245J9.4',\n",
       " 'ENSG00000144746': 'ARL6IP5',\n",
       " 'ENSG00000163380': 'LMOD3',\n",
       " 'ENSG00000114541': 'FRMD4B',\n",
       " 'ENSG00000270562': 'RP11-154H23.3',\n",
       " 'ENSG00000241163': 'LINC00877',\n",
       " 'ENSG00000172969': 'FRG2C',\n",
       " 'ENSG00000169855': 'ROBO1',\n",
       " 'ENSG00000175161': 'CADM2',\n",
       " 'ENSG00000184500': 'PROS1',\n",
       " 'ENSG00000064225': 'ST3GAL6',\n",
       " 'ENSG00000144810': 'COL8A1',\n",
       " 'ENSG00000168386': 'FILIP1L',\n",
       " 'ENSG00000154175': 'ABI3BP',\n",
       " 'ENSG00000081148': 'IMPG2',\n",
       " 'ENSG00000144815': 'NXPE3',\n",
       " 'ENSG00000144802': 'NFKBIZ',\n",
       " 'ENSG00000170017': 'ALCAM',\n",
       " 'ENSG00000114423': 'CBLB',\n",
       " 'ENSG00000239828': 'RP11-446H18.5',\n",
       " 'ENSG00000273125': 'RP11-115H18.1',\n",
       " 'ENSG00000241469': 'LINC00635',\n",
       " 'ENSG00000196776': 'CD47',\n",
       " 'ENSG00000144821': 'MYH15',\n",
       " 'ENSG00000228980': 'LINC01205',\n",
       " 'ENSG00000206532': 'RP11-553A10.1',\n",
       " 'ENSG00000239482': 'RP11-90K6.1',\n",
       " 'ENSG00000243081': 'RP11-231E6.1',\n",
       " ...}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "map_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5ccf18b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>You are an expert in single-cell biology and f...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>You are a domain expert in functional genomics...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>You are assisting with the interpretation of p...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>You are an expert in regulatory genomics. Cons...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>You are a specialist in gene perturbation expe...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3145</th>\n",
       "      <td>You are an expert in gene expression analysis....</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3146</th>\n",
       "      <td>You are a genomics expert evaluating perturbat...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3147</th>\n",
       "      <td>You are an expert analyst of perturbation data...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3148</th>\n",
       "      <td>You are a functional genomics specialist. In K...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3149</th>\n",
       "      <td>You are an expert in gene regulation studies. ...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3150 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               question answer\n",
       "0     You are an expert in single-cell biology and f...    Yes\n",
       "1     You are a domain expert in functional genomics...    Yes\n",
       "2     You are assisting with the interpretation of p...    Yes\n",
       "3     You are an expert in regulatory genomics. Cons...    Yes\n",
       "4     You are a specialist in gene perturbation expe...    Yes\n",
       "...                                                 ...    ...\n",
       "3145  You are an expert in gene expression analysis....    Yes\n",
       "3146  You are a genomics expert evaluating perturbat...    Yes\n",
       "3147  You are an expert analyst of perturbation data...    Yes\n",
       "3148  You are a functional genomics specialist. In K...    Yes\n",
       "3149  You are an expert in gene regulation studies. ...    Yes\n",
       "\n",
       "[3150 rows x 2 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ae02062f",
   "metadata": {},
   "outputs": [],
   "source": [
    "que_new = []\n",
    "for item in df['question'].values:\n",
    "    for j in map_dict.keys():\n",
    "        if j in item:\n",
    "            item_new = item.replace(j, map_dict[j])\n",
    "            que_new.append(item_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8a3ac9b4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3150"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(que_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "84efe73e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['question_new'] = que_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c2c98374",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.model_selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1f2b89ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>question_new</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>You are an expert in single-cell biology and f...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are an expert in single-cell biology and f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>You are a domain expert in functional genomics...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are a domain expert in functional genomics...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>You are assisting with the interpretation of p...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are assisting with the interpretation of p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>You are an expert in regulatory genomics. Cons...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are an expert in regulatory genomics. Cons...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>You are a specialist in gene perturbation expe...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are a specialist in gene perturbation expe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3145</th>\n",
       "      <td>You are an expert in gene expression analysis....</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are an expert in gene expression analysis....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3146</th>\n",
       "      <td>You are a genomics expert evaluating perturbat...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are a genomics expert evaluating perturbat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3147</th>\n",
       "      <td>You are an expert analyst of perturbation data...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are an expert analyst of perturbation data...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3148</th>\n",
       "      <td>You are a functional genomics specialist. In K...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are a functional genomics specialist. In K...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3149</th>\n",
       "      <td>You are an expert in gene regulation studies. ...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are an expert in gene regulation studies. ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3150 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               question answer  \\\n",
       "0     You are an expert in single-cell biology and f...    Yes   \n",
       "1     You are a domain expert in functional genomics...    Yes   \n",
       "2     You are assisting with the interpretation of p...    Yes   \n",
       "3     You are an expert in regulatory genomics. Cons...    Yes   \n",
       "4     You are a specialist in gene perturbation expe...    Yes   \n",
       "...                                                 ...    ...   \n",
       "3145  You are an expert in gene expression analysis....    Yes   \n",
       "3146  You are a genomics expert evaluating perturbat...    Yes   \n",
       "3147  You are an expert analyst of perturbation data...    Yes   \n",
       "3148  You are a functional genomics specialist. In K...    Yes   \n",
       "3149  You are an expert in gene regulation studies. ...    Yes   \n",
       "\n",
       "                                           question_new  \n",
       "0     You are an expert in single-cell biology and f...  \n",
       "1     You are a domain expert in functional genomics...  \n",
       "2     You are assisting with the interpretation of p...  \n",
       "3     You are an expert in regulatory genomics. Cons...  \n",
       "4     You are a specialist in gene perturbation expe...  \n",
       "...                                                 ...  \n",
       "3145  You are an expert in gene expression analysis....  \n",
       "3146  You are a genomics expert evaluating perturbat...  \n",
       "3147  You are an expert analyst of perturbation data...  \n",
       "3148  You are a functional genomics specialist. In K...  \n",
       "3149  You are an expert in gene regulation studies. ...  \n",
       "\n",
       "[3150 rows x 3 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "88656b2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_id, test_id = sklearn.model_selection.train_test_split(list(df.index), test_size=3000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e3c44c80",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "answer\n",
       "Yes    141\n",
       "No       9\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[train_id]['answer'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e9206b34",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[train_id].to_csv(\"norman_train.csv\")\n",
    "df.loc[test_id].to_csv(\"norman_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "af9949c1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3000"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df.loc[test_id])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "7beca8ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "# /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/adamson_train.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "b30e6e5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_d = df.loc[train_id]\n",
    "train_d['_id'] = list(train_d.index)\n",
    "\n",
    "test_d = df.loc[test_id]\n",
    "test_d['_id'] = list(test_d.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "15574dfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_d.to_json(\"./norman_train.json\", orient='records', indent=2)\n",
    "test_d.to_json(\"./norman_test.json\", orient='records', indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca11fa03",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "a49b6c89",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "e02e5c21",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"/gpfs/radev/project/zhao/hs2234/evox/QA/gene_perturb/qa/replogle_psa_single_qa.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "82e1db47",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"You are an expert in single-cell biology and functional genomics. In K562 cells, AAAS is perturbed and the expression of AAAS is measured. Does this perturbation cause a significant change in AAAS expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\""
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['question'].values[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "4ba277c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "answer\n",
       "Yes    51540\n",
       "No     10530\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['answer'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "04ce9348",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/anndata/_core/anndata.py:1808: UserWarning: Variable names are not unique. To make them unique, call `.var_names_make_unique`.\n",
      "  utils.warn_names_duplicates(\"var\")\n"
     ]
    }
   ],
   "source": [
    "import scanpy as sc\n",
    "adata = sc.read_h5ad(\"/home/tl688/pitl688/pert_test/data/reploge/perturb_processed.h5ad\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "b0d4c7cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# adata.var\n",
    "map_dict = {}\n",
    "for i in adata.var.index:\n",
    "    gene = adata.var.loc[i]['gene_name']\n",
    "    map_dict[i] = gene"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "2f27e181",
   "metadata": {},
   "outputs": [],
   "source": [
    "# more powerful model;\n",
    "# api/knowledge base query;\n",
    "# analyze system; "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "6aee5c81",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'NOC2L': 'NOC2L',\n",
       " 'PLEKHN1': 'PLEKHN1',\n",
       " 'HES4': 'HES4',\n",
       " 'ISG15': 'ISG15',\n",
       " 'AGRN': 'AGRN',\n",
       " 'SDF4': 'SDF4',\n",
       " 'B3GALT6': 'B3GALT6',\n",
       " 'UBE2J2': 'UBE2J2',\n",
       " 'ACAP3': 'ACAP3',\n",
       " 'PUSL1': 'PUSL1',\n",
       " 'INTS11': 'INTS11',\n",
       " 'CPTP': 'CPTP',\n",
       " 'DVL1': 'DVL1',\n",
       " 'MXRA8': 'MXRA8',\n",
       " 'AURKAIP1': 'AURKAIP1',\n",
       " 'CCNL2': 'CCNL2',\n",
       " 'MRPL20-AS1': 'MRPL20-AS1',\n",
       " 'MRPL20': 'MRPL20',\n",
       " 'ATAD3B': 'ATAD3B',\n",
       " 'ATAD3A': 'ATAD3A',\n",
       " 'SSU72': 'SSU72',\n",
       " 'MIB2': 'MIB2',\n",
       " 'CDK11B': 'CDK11B',\n",
       " 'CDK11A': 'CDK11A',\n",
       " 'NADK': 'NADK',\n",
       " 'GNB1': 'GNB1',\n",
       " 'FAAP20': 'FAAP20',\n",
       " 'SKI': 'SKI',\n",
       " 'RER1': 'RER1',\n",
       " 'PEX10': 'PEX10',\n",
       " 'PRXL2B': 'PRXL2B',\n",
       " 'MEGF6': 'MEGF6',\n",
       " 'TPRG1L': 'TPRG1L',\n",
       " 'WRAP73': 'WRAP73',\n",
       " 'LRRC47': 'LRRC47',\n",
       " 'CEP104': 'CEP104',\n",
       " 'C1orf174': 'C1orf174',\n",
       " 'RPL22': 'RPL22',\n",
       " 'RNF207': 'RNF207',\n",
       " 'ICMT': 'ICMT',\n",
       " 'ACOT7': 'ACOT7',\n",
       " 'NOL9': 'NOL9',\n",
       " 'KLHL21': 'KLHL21',\n",
       " 'PHF13': 'PHF13',\n",
       " 'THAP3': 'THAP3',\n",
       " 'DNAJC11': 'DNAJC11',\n",
       " 'CAMTA1': 'CAMTA1',\n",
       " 'VAMP3': 'VAMP3',\n",
       " 'PER3': 'PER3',\n",
       " 'PARK7': 'PARK7',\n",
       " 'ERRFI1': 'ERRFI1',\n",
       " 'RERE': 'RERE',\n",
       " 'ENO1': 'ENO1',\n",
       " 'SPSB1': 'SPSB1',\n",
       " 'SLC25A33': 'SLC25A33',\n",
       " 'CLSTN1': 'CLSTN1',\n",
       " 'CTNNBIP1': 'CTNNBIP1',\n",
       " 'LZIC': 'LZIC',\n",
       " 'NMNAT1': 'NMNAT1',\n",
       " 'UBE4B': 'UBE4B',\n",
       " 'KIF1B': 'KIF1B',\n",
       " 'PGD': 'PGD',\n",
       " 'CENPS': 'CENPS',\n",
       " 'DFFA': 'DFFA',\n",
       " 'PEX14': 'PEX14',\n",
       " 'TARDBP': 'TARDBP',\n",
       " 'SRM': 'SRM',\n",
       " 'EXOSC10': 'EXOSC10',\n",
       " 'MTOR': 'MTOR',\n",
       " 'UBIAD1': 'UBIAD1',\n",
       " 'MAD2L2': 'MAD2L2',\n",
       " 'AGTRAP': 'AGTRAP',\n",
       " 'KIAA2013': 'KIAA2013',\n",
       " 'PLOD1': 'PLOD1',\n",
       " 'MFN2': 'MFN2',\n",
       " 'MIIP': 'MIIP',\n",
       " 'VPS13D': 'VPS13D',\n",
       " 'DHRS3': 'DHRS3',\n",
       " 'PRDM2': 'PRDM2',\n",
       " 'EFHD2': 'EFHD2',\n",
       " 'DNAJC16': 'DNAJC16',\n",
       " 'DDI2': 'DDI2',\n",
       " 'PLEKHM2': 'PLEKHM2',\n",
       " 'FBLIM1': 'FBLIM1',\n",
       " 'SPEN': 'SPEN',\n",
       " 'ZBTB17': 'ZBTB17',\n",
       " 'EPHA2': 'EPHA2',\n",
       " 'FBXO42': 'FBXO42',\n",
       " 'SZRD1': 'SZRD1',\n",
       " 'NECAP2': 'NECAP2',\n",
       " 'NBPF1': 'NBPF1',\n",
       " 'MFAP2': 'MFAP2',\n",
       " 'ATP13A2': 'ATP13A2',\n",
       " 'SDHB': 'SDHB',\n",
       " 'RCC2': 'RCC2',\n",
       " 'IFFO2': 'IFFO2',\n",
       " 'UBR4': 'UBR4',\n",
       " 'EMC1': 'EMC1',\n",
       " 'MRTO4': 'MRTO4',\n",
       " 'AKR7A2': 'AKR7A2',\n",
       " 'SLC66A1': 'SLC66A1',\n",
       " 'CAPZB': 'CAPZB',\n",
       " 'MICOS10': 'MICOS10',\n",
       " 'OTUD3': 'OTUD3',\n",
       " 'CAMK2N1': 'CAMK2N1',\n",
       " 'MUL1': 'MUL1',\n",
       " 'PINK1': 'PINK1',\n",
       " 'DDOST': 'DDOST',\n",
       " 'HP1BP3': 'HP1BP3',\n",
       " 'EIF4G3': 'EIF4G3',\n",
       " 'ECE1': 'ECE1',\n",
       " 'USP48': 'USP48',\n",
       " 'HSPG2': 'HSPG2',\n",
       " 'CDC42': 'CDC42',\n",
       " 'ZBTB40': 'ZBTB40',\n",
       " 'EPHB2': 'EPHB2',\n",
       " 'KDM1A': 'KDM1A',\n",
       " 'LUZP1': 'LUZP1',\n",
       " 'HNRNPR': 'HNRNPR',\n",
       " 'ID3': 'ID3',\n",
       " 'RPL11': 'RPL11',\n",
       " 'ELOA': 'ELOA',\n",
       " 'PITHD1': 'PITHD1',\n",
       " 'LYPLA2': 'LYPLA2',\n",
       " 'GALE': 'GALE',\n",
       " 'HMGCL': 'HMGCL',\n",
       " 'FUCA1': 'FUCA1',\n",
       " 'PNRC2': 'PNRC2',\n",
       " 'SRSF10': 'SRSF10',\n",
       " 'STPG1': 'STPG1',\n",
       " 'NIPAL3': 'NIPAL3',\n",
       " 'RCAN3': 'RCAN3',\n",
       " 'SRRM1': 'SRRM1',\n",
       " 'CLIC4': 'CLIC4',\n",
       " 'SYF2': 'SYF2',\n",
       " 'RSRP1': 'RSRP1',\n",
       " 'TMEM50A': 'TMEM50A',\n",
       " 'MACO1': 'MACO1',\n",
       " 'SELENON': 'SELENON',\n",
       " 'MTFR1L': 'MTFR1L',\n",
       " 'STMN1': 'STMN1',\n",
       " 'ZNF593': 'ZNF593',\n",
       " 'SH3BGRL3': 'SH3BGRL3',\n",
       " 'DHDDS': 'DHDDS',\n",
       " 'HMGN2': 'HMGN2',\n",
       " 'ARID1A': 'ARID1A',\n",
       " 'PIGV': 'PIGV',\n",
       " 'GPN2': 'GPN2',\n",
       " 'NUDC': 'NUDC',\n",
       " 'TRNP1': 'TRNP1',\n",
       " 'SLC9A1': 'SLC9A1',\n",
       " 'WDTC1': 'WDTC1',\n",
       " 'TMEM222': 'TMEM222',\n",
       " 'WASF2': 'WASF2',\n",
       " 'AHDC1': 'AHDC1',\n",
       " 'IFI6': 'IFI6',\n",
       " 'FAM76A': 'FAM76A',\n",
       " 'STX12': 'STX12',\n",
       " 'PPP1R8': 'PPP1R8',\n",
       " 'RPA2': 'RPA2',\n",
       " 'XKR8': 'XKR8',\n",
       " 'EYA3': 'EYA3',\n",
       " 'DNAJC8': 'DNAJC8',\n",
       " 'ATP5IF1': 'ATP5IF1',\n",
       " 'SESN2': 'SESN2',\n",
       " 'PHACTR4': 'PHACTR4',\n",
       " 'RCC1': 'RCC1',\n",
       " 'SNHG3': 'SNHG3',\n",
       " 'TRNAU1AP': 'TRNAU1AP',\n",
       " 'SNHG12': 'SNHG12',\n",
       " 'TAF12': 'TAF12',\n",
       " 'GMEB1': 'GMEB1',\n",
       " 'YTHDF2': 'YTHDF2',\n",
       " 'EPB41': 'EPB41',\n",
       " 'SRSF4': 'SRSF4',\n",
       " 'MECR': 'MECR',\n",
       " 'PUM1': 'PUM1',\n",
       " 'SNRNP40': 'SNRNP40',\n",
       " 'ZCCHC17': 'ZCCHC17',\n",
       " 'SERINC2': 'SERINC2',\n",
       " 'PEF1': 'PEF1',\n",
       " 'SPOCD1': 'SPOCD1',\n",
       " 'PTP4A2': 'PTP4A2',\n",
       " 'KHDRBS1': 'KHDRBS1',\n",
       " 'TMEM39B': 'TMEM39B',\n",
       " 'KPNA6': 'KPNA6',\n",
       " 'TXLNA': 'TXLNA',\n",
       " 'CCDC28B': 'CCDC28B',\n",
       " 'TMEM234': 'TMEM234',\n",
       " 'EIF3I': 'EIF3I',\n",
       " 'HDAC1': 'HDAC1',\n",
       " 'MARCKSL1': 'MARCKSL1',\n",
       " 'BSDC1': 'BSDC1',\n",
       " 'ZBTB8A': 'ZBTB8A',\n",
       " 'ZBTB8OS': 'ZBTB8OS',\n",
       " 'RBBP4': 'RBBP4',\n",
       " 'SYNC': 'SYNC',\n",
       " 'KIAA1522': 'KIAA1522',\n",
       " 'YARS': 'YARS',\n",
       " 'S100PBP': 'S100PBP',\n",
       " 'TMEM54': 'TMEM54',\n",
       " 'AK2': 'AK2',\n",
       " 'PHC2': 'PHC2',\n",
       " 'SMIM12': 'SMIM12',\n",
       " 'TMEM35B': 'TMEM35B',\n",
       " 'ZMYM6': 'ZMYM6',\n",
       " 'ZMYM1': 'ZMYM1',\n",
       " 'SFPQ': 'SFPQ',\n",
       " 'ZMYM4': 'ZMYM4',\n",
       " 'KIAA0319L': 'KIAA0319L',\n",
       " 'NCDN': 'NCDN',\n",
       " 'PSMB2': 'PSMB2',\n",
       " 'C1orf216': 'C1orf216',\n",
       " 'CLSPN': 'CLSPN',\n",
       " 'AGO1': 'AGO1',\n",
       " 'AGO3': 'AGO3',\n",
       " 'ADPRHL2': 'ADPRHL2',\n",
       " 'TRAPPC3': 'TRAPPC3',\n",
       " 'MAP7D1': 'MAP7D1',\n",
       " 'THRAP3': 'THRAP3',\n",
       " 'SH3D21': 'SH3D21',\n",
       " 'EVA1B': 'EVA1B',\n",
       " 'STK40': 'STK40',\n",
       " 'LSM10': 'LSM10',\n",
       " 'MRPS15': 'MRPS15',\n",
       " 'MEAF6': 'MEAF6',\n",
       " 'SNIP1': 'SNIP1',\n",
       " 'DNALI1': 'DNALI1',\n",
       " 'GNL2': 'GNL2',\n",
       " 'C1orf109': 'C1orf109',\n",
       " 'CDCA8': 'CDCA8',\n",
       " 'MANEAL': 'MANEAL',\n",
       " 'YRDC': 'YRDC',\n",
       " 'C1orf122': 'C1orf122',\n",
       " 'MTF1': 'MTF1',\n",
       " 'SF3A3': 'SF3A3',\n",
       " 'FHL3': 'FHL3',\n",
       " 'UTP11': 'UTP11',\n",
       " 'RRAGC': 'RRAGC',\n",
       " 'MYCBP': 'MYCBP',\n",
       " 'AKIRIN1': 'AKIRIN1',\n",
       " 'NDUFS5': 'NDUFS5',\n",
       " 'MACF1': 'MACF1',\n",
       " 'PABPC4': 'PABPC4',\n",
       " 'PPIE': 'PPIE',\n",
       " 'TRIT1': 'TRIT1',\n",
       " 'CAP1': 'CAP1',\n",
       " 'PPT1': 'PPT1',\n",
       " 'RLF': 'RLF',\n",
       " 'ZMPSTE24': 'ZMPSTE24',\n",
       " 'SMAP2': 'SMAP2',\n",
       " 'NFYC': 'NFYC',\n",
       " 'CITED4': 'CITED4',\n",
       " 'CTPS1': 'CTPS1',\n",
       " 'SCMH1': 'SCMH1',\n",
       " 'FOXJ3': 'FOXJ3',\n",
       " 'PPCS': 'PPCS',\n",
       " 'PPIH': 'PPIH',\n",
       " 'YBX1': 'YBX1',\n",
       " 'P3H1': 'P3H1',\n",
       " 'C1orf50': 'C1orf50',\n",
       " 'SVBP': 'SVBP',\n",
       " 'SLC2A1': 'SLC2A1',\n",
       " 'EBNA1BP2': 'EBNA1BP2',\n",
       " 'CDC20': 'CDC20',\n",
       " 'ELOVL1': 'ELOVL1',\n",
       " 'MED8': 'MED8',\n",
       " 'SZT2': 'SZT2',\n",
       " 'HYI': 'HYI',\n",
       " 'PTPRF': 'PTPRF',\n",
       " 'KDM4A': 'KDM4A',\n",
       " 'ST3GAL3': 'ST3GAL3',\n",
       " 'IPO13': 'IPO13',\n",
       " 'DPH2': 'DPH2',\n",
       " 'ATP6V0B': 'ATP6V0B',\n",
       " 'B4GALT2': 'B4GALT2',\n",
       " 'DMAP1': 'DMAP1',\n",
       " 'ERI3': 'ERI3',\n",
       " 'RNF220': 'RNF220',\n",
       " 'KIF2C': 'KIF2C',\n",
       " 'RPS8': 'RPS8',\n",
       " 'PLK3': 'PLK3',\n",
       " 'EIF2B3': 'EIF2B3',\n",
       " 'UROD': 'UROD',\n",
       " 'TOE1': 'TOE1',\n",
       " 'PRDX1': 'PRDX1',\n",
       " 'AKR1A1': 'AKR1A1',\n",
       " 'NASP': 'NASP',\n",
       " 'GPBP1L1': 'GPBP1L1',\n",
       " 'TMEM69': 'TMEM69',\n",
       " 'IPP': 'IPP',\n",
       " 'MAST2': 'MAST2',\n",
       " 'PIK3R3': 'PIK3R3',\n",
       " 'POMGNT1': 'POMGNT1',\n",
       " 'LRRC41': 'LRRC41',\n",
       " 'UQCRH': 'UQCRH',\n",
       " 'NSUN4': 'NSUN4',\n",
       " 'MKNK1': 'MKNK1',\n",
       " 'ATPAF1': 'ATPAF1',\n",
       " 'EFCAB14': 'EFCAB14',\n",
       " 'STIL': 'STIL',\n",
       " 'CMPK1': 'CMPK1',\n",
       " 'FAF1': 'FAF1',\n",
       " 'CDKN2C': 'CDKN2C',\n",
       " 'RNF11': 'RNF11',\n",
       " 'EPS15': 'EPS15',\n",
       " 'OSBPL9': 'OSBPL9',\n",
       " 'NRDC': 'NRDC',\n",
       " 'RAB3B': 'RAB3B',\n",
       " 'TXNDC12': 'TXNDC12',\n",
       " 'KTI12': 'KTI12',\n",
       " 'BTF3L4': 'BTF3L4',\n",
       " 'ZFYVE9': 'ZFYVE9',\n",
       " 'CC2D1B': 'CC2D1B',\n",
       " 'ORC1': 'ORC1',\n",
       " 'PRPF38A': 'PRPF38A',\n",
       " 'TUT4': 'TUT4',\n",
       " 'COA7': 'COA7',\n",
       " 'ZYG11B': 'ZYG11B',\n",
       " 'ECHDC2': 'ECHDC2',\n",
       " 'SCP2': 'SCP2',\n",
       " 'CPT2': 'CPT2',\n",
       " 'CZIB': 'CZIB',\n",
       " 'MAGOH': 'MAGOH',\n",
       " 'LRP8': 'LRP8',\n",
       " 'NDC1': 'NDC1',\n",
       " 'YIPF1': 'YIPF1',\n",
       " 'HSPB11': 'HSPB11',\n",
       " 'LRRC42': 'LRRC42',\n",
       " 'TMEM59': 'TMEM59',\n",
       " 'MRPL37': 'MRPL37',\n",
       " 'SSBP3': 'SSBP3',\n",
       " 'TTC4': 'TTC4',\n",
       " 'DHCR24': 'DHCR24',\n",
       " 'USP24': 'USP24',\n",
       " 'PLPP3': 'PLPP3',\n",
       " 'PRKAA2': 'PRKAA2',\n",
       " 'MYSM1': 'MYSM1',\n",
       " 'JUN': 'JUN',\n",
       " 'NFIA': 'NFIA',\n",
       " 'TM2D1': 'TM2D1',\n",
       " 'PATJ': 'PATJ',\n",
       " 'USP1': 'USP1',\n",
       " 'DOCK7': 'DOCK7',\n",
       " 'ATG4C': 'ATG4C',\n",
       " 'ITGB3BP': 'ITGB3BP',\n",
       " 'PGM1': 'PGM1',\n",
       " 'ROR1': 'ROR1',\n",
       " 'JAK1': 'JAK1',\n",
       " 'AK4': 'AK4',\n",
       " 'LEPROT': 'LEPROT',\n",
       " 'SGIP1': 'SGIP1',\n",
       " 'MIER1': 'MIER1',\n",
       " 'SLC35D1': 'SLC35D1',\n",
       " 'SERBP1': 'SERBP1',\n",
       " 'GADD45A': 'GADD45A',\n",
       " 'GNG12': 'GNG12',\n",
       " 'WLS': 'WLS',\n",
       " 'DEPDC1': 'DEPDC1',\n",
       " 'LRRC40': 'LRRC40',\n",
       " 'SRSF11': 'SRSF11',\n",
       " 'ANKRD13C': 'ANKRD13C',\n",
       " 'ZRANB2': 'ZRANB2',\n",
       " 'NEGR1': 'NEGR1',\n",
       " 'CRYZ': 'CRYZ',\n",
       " 'TYW3': 'TYW3',\n",
       " 'ACADM': 'ACADM',\n",
       " 'RABGGTB': 'RABGGTB',\n",
       " 'PIGK': 'PIGK',\n",
       " 'AK5': 'AK5',\n",
       " 'AC118549.1': 'AC118549.1',\n",
       " 'USP33': 'USP33',\n",
       " 'MIGA1': 'MIGA1',\n",
       " 'NEXN': 'NEXN',\n",
       " 'FUBP1': 'FUBP1',\n",
       " 'DNAJB4': 'DNAJB4',\n",
       " 'ADGRL4': 'ADGRL4',\n",
       " 'TTLL7': 'TTLL7',\n",
       " 'PRKACB': 'PRKACB',\n",
       " 'RPF1': 'RPF1',\n",
       " 'GNG5': 'GNG5',\n",
       " 'CTBS': 'CTBS',\n",
       " 'SSX2IP': 'SSX2IP',\n",
       " 'C1orf52': 'C1orf52',\n",
       " 'BCL10': 'BCL10',\n",
       " 'DDAH1': 'DDAH1',\n",
       " 'AC092807.3': 'AC092807.3',\n",
       " 'CCN1': 'CCN1',\n",
       " 'ZNHIT6': 'ZNHIT6',\n",
       " 'ODF2L': 'ODF2L',\n",
       " 'SH3GLB1': 'SH3GLB1',\n",
       " 'SELENOF': 'SELENOF',\n",
       " 'HS2ST1': 'HS2ST1',\n",
       " 'LMO4': 'LMO4',\n",
       " 'PKN2': 'PKN2',\n",
       " 'GTF2B': 'GTF2B',\n",
       " 'KYAT3': 'KYAT3',\n",
       " 'RBMXL1': 'RBMXL1',\n",
       " 'GBP3': 'GBP3',\n",
       " 'GBP1': 'GBP1',\n",
       " 'LRRC8C': 'LRRC8C',\n",
       " 'LRRC8D': 'LRRC8D',\n",
       " 'ZNF326': 'ZNF326',\n",
       " 'ZNF644': 'ZNF644',\n",
       " 'EPHX4': 'EPHX4',\n",
       " 'GLMN': 'GLMN',\n",
       " 'RPAP2': 'RPAP2',\n",
       " 'EVI5': 'EVI5',\n",
       " 'RPL5': 'RPL5',\n",
       " 'DIPK1A': 'DIPK1A',\n",
       " 'MTF2': 'MTF2',\n",
       " 'TMED5': 'TMED5',\n",
       " 'CCDC18': 'CCDC18',\n",
       " 'CCDC18-AS1': 'CCDC18-AS1',\n",
       " 'DR1': 'DR1',\n",
       " 'FNBP1L': 'FNBP1L',\n",
       " 'BCAR3': 'BCAR3',\n",
       " 'DNTTIP2': 'DNTTIP2',\n",
       " 'GCLM': 'GCLM',\n",
       " 'ARHGAP29': 'ARHGAP29',\n",
       " 'ABCD3': 'ABCD3',\n",
       " 'F3': 'F3',\n",
       " 'CNN3': 'CNN3',\n",
       " 'ALG14': 'ALG14',\n",
       " 'RWDD3': 'RWDD3',\n",
       " 'PTBP2': 'PTBP2',\n",
       " 'DPYD': 'DPYD',\n",
       " 'MIR137HG': 'MIR137HG',\n",
       " 'SNX7': 'SNX7',\n",
       " 'PALMD': 'PALMD',\n",
       " 'AGL': 'AGL',\n",
       " 'SLC35A3': 'SLC35A3',\n",
       " 'MFSD14A': 'MFSD14A',\n",
       " 'SASS6': 'SASS6',\n",
       " 'TRMT13': 'TRMT13',\n",
       " 'DBT': 'DBT',\n",
       " 'RTCA': 'RTCA',\n",
       " 'EXTL2': 'EXTL2',\n",
       " 'SLC30A7': 'SLC30A7',\n",
       " 'DPH5': 'DPH5',\n",
       " 'AC093157.1': 'AC093157.1',\n",
       " 'RNPC3': 'RNPC3',\n",
       " 'PRMT6': 'PRMT6',\n",
       " 'NTNG1': 'NTNG1',\n",
       " 'SLC25A24': 'SLC25A24',\n",
       " 'PRPF38B': 'PRPF38B',\n",
       " 'STXBP3': 'STXBP3',\n",
       " 'GPSM2': 'GPSM2',\n",
       " 'CLCC1': 'CLCC1',\n",
       " 'WDR47': 'WDR47',\n",
       " 'TAF13': 'TAF13',\n",
       " 'TMEM167B': 'TMEM167B',\n",
       " 'SARS': 'SARS',\n",
       " 'PSRC1': 'PSRC1',\n",
       " 'SORT1': 'SORT1',\n",
       " 'PSMA5': 'PSMA5',\n",
       " 'GNAI3': 'GNAI3',\n",
       " 'AMPD2': 'AMPD2',\n",
       " 'GSTM3': 'GSTM3',\n",
       " 'CSF1': 'CSF1',\n",
       " 'AHCYL1': 'AHCYL1',\n",
       " 'STRIP1': 'STRIP1',\n",
       " 'SLC16A4': 'SLC16A4',\n",
       " 'LAMTOR5': 'LAMTOR5',\n",
       " 'LRIF1': 'LRIF1',\n",
       " 'DRAM2': 'DRAM2',\n",
       " 'CEPT1': 'CEPT1',\n",
       " 'WDR77': 'WDR77',\n",
       " 'ATP5PB': 'ATP5PB',\n",
       " 'RAP1A': 'RAP1A',\n",
       " 'DDX20': 'DDX20',\n",
       " 'CTTNBP2NL': 'CTTNBP2NL',\n",
       " 'ST7L': 'ST7L',\n",
       " 'CAPZA1': 'CAPZA1',\n",
       " 'MOV10': 'MOV10',\n",
       " 'RHOC': 'RHOC',\n",
       " 'SLC16A1': 'SLC16A1',\n",
       " 'SLC16A1-AS1': 'SLC16A1-AS1',\n",
       " 'LRIG2': 'LRIG2',\n",
       " 'MAGI3': 'MAGI3',\n",
       " 'PHTF1': 'PHTF1',\n",
       " 'RSBN1': 'RSBN1',\n",
       " 'HIPK1': 'HIPK1',\n",
       " 'TRIM33': 'TRIM33',\n",
       " 'BCAS2': 'BCAS2',\n",
       " 'NRAS': 'NRAS',\n",
       " 'CSDE1': 'CSDE1',\n",
       " 'SIKE1': 'SIKE1',\n",
       " 'NGF': 'NGF',\n",
       " 'VANGL1': 'VANGL1',\n",
       " 'ATP1A1': 'ATP1A1',\n",
       " 'CD58': 'CD58',\n",
       " 'TTF2': 'TTF2',\n",
       " 'MAN1A2': 'MAN1A2',\n",
       " 'GDAP2': 'GDAP2',\n",
       " 'WDR3': 'WDR3',\n",
       " 'WARS2': 'WARS2',\n",
       " 'PHGDH': 'PHGDH',\n",
       " 'NOTCH2': 'NOTCH2',\n",
       " 'SEC22B': 'SEC22B',\n",
       " 'SRGAP2C': 'SRGAP2C',\n",
       " 'LINC02802': 'LINC02802',\n",
       " 'SRGAP2B': 'SRGAP2B',\n",
       " 'RNF115': 'RNF115',\n",
       " 'POLR3C': 'POLR3C',\n",
       " 'PIAS3': 'PIAS3',\n",
       " 'PEX11B': 'PEX11B',\n",
       " 'RBM8A': 'RBM8A',\n",
       " 'LIX1L': 'LIX1L',\n",
       " 'POLR3GL': 'POLR3GL',\n",
       " 'TXNIP': 'TXNIP',\n",
       " 'NBPF10': 'NBPF10',\n",
       " 'NOTCH2NLA': 'NOTCH2NLA',\n",
       " 'PRKAB2': 'PRKAB2',\n",
       " 'NBPF14': 'NBPF14',\n",
       " 'PDE4DIP': 'PDE4DIP',\n",
       " 'AC245297.3': 'AC245297.3',\n",
       " 'HIST2H2AC': 'HIST2H2AC',\n",
       " 'BOLA1': 'BOLA1',\n",
       " 'SV2A': 'SV2A',\n",
       " 'SF3B4': 'SF3B4',\n",
       " 'OTUD7B': 'OTUD7B',\n",
       " 'VPS45': 'VPS45',\n",
       " 'PLEKHO1': 'PLEKHO1',\n",
       " 'ANP32E': 'ANP32E',\n",
       " 'APH1A': 'APH1A',\n",
       " 'MRPS21': 'MRPS21',\n",
       " 'PRPF3': 'PRPF3',\n",
       " 'RPRD2': 'RPRD2',\n",
       " 'TARS2': 'TARS2',\n",
       " 'ECM1': 'ECM1',\n",
       " 'MCL1': 'MCL1',\n",
       " 'ENSA': 'ENSA',\n",
       " 'GOLPH3L': 'GOLPH3L',\n",
       " 'CTSK': 'CTSK',\n",
       " 'ARNT': 'ARNT',\n",
       " 'SETDB1': 'SETDB1',\n",
       " 'CERS2': 'CERS2',\n",
       " 'C1orf56': 'C1orf56',\n",
       " 'CDC42SE1': 'CDC42SE1',\n",
       " 'MLLT11': 'MLLT11',\n",
       " 'GABPB2': 'GABPB2',\n",
       " 'SCNM1': 'SCNM1',\n",
       " 'VPS72': 'VPS72',\n",
       " 'PIP5K1A': 'PIP5K1A',\n",
       " 'PSMD4': 'PSMD4',\n",
       " 'PI4KB': 'PI4KB',\n",
       " 'RFX5': 'RFX5',\n",
       " 'PSMB4': 'PSMB4',\n",
       " 'POGZ': 'POGZ',\n",
       " 'TUFT1': 'TUFT1',\n",
       " 'SNX27': 'SNX27',\n",
       " 'MRPL9': 'MRPL9',\n",
       " 'THEM4': 'THEM4',\n",
       " 'S100A10': 'S100A10',\n",
       " 'S100A11': 'S100A11',\n",
       " 'S100A6': 'S100A6',\n",
       " 'S100A2': 'S100A2',\n",
       " 'S100A16': 'S100A16',\n",
       " 'S100A13': 'S100A13',\n",
       " 'CHTOP': 'CHTOP',\n",
       " 'SNAPIN': 'SNAPIN',\n",
       " 'ILF2': 'ILF2',\n",
       " 'INTS3': 'INTS3',\n",
       " 'GATAD2B': 'GATAD2B',\n",
       " 'DENND4B': 'DENND4B',\n",
       " 'SLC39A1': 'SLC39A1',\n",
       " 'JTB': 'JTB',\n",
       " 'RAB13': 'RAB13',\n",
       " 'RPS27': 'RPS27',\n",
       " 'TPM3': 'TPM3',\n",
       " 'C1orf43': 'C1orf43',\n",
       " 'UBAP2L': 'UBAP2L',\n",
       " 'HAX1': 'HAX1',\n",
       " 'ATP8B2': 'ATP8B2',\n",
       " 'UBE2Q1': 'UBE2Q1',\n",
       " 'ADAR': 'ADAR',\n",
       " 'PMVK': 'PMVK',\n",
       " 'PBXIP1': 'PBXIP1',\n",
       " 'SHC1': 'SHC1',\n",
       " 'CKS1B': 'CKS1B',\n",
       " 'FLAD1': 'FLAD1',\n",
       " 'ADAM15': 'ADAM15',\n",
       " 'EFNA4': 'EFNA4',\n",
       " 'SLC50A1': 'SLC50A1',\n",
       " 'DPM3': 'DPM3',\n",
       " 'KRTCAP2': 'KRTCAP2',\n",
       " 'MTX1': 'MTX1',\n",
       " 'GBA': 'GBA',\n",
       " 'FAM189B': 'FAM189B',\n",
       " 'SCAMP3': 'SCAMP3',\n",
       " 'FDPS': 'FDPS',\n",
       " 'RUSC1': 'RUSC1',\n",
       " 'ASH1L': 'ASH1L',\n",
       " 'MSTO1': 'MSTO1',\n",
       " 'YY1AP1': 'YY1AP1',\n",
       " 'DAP3': 'DAP3',\n",
       " 'GON4L': 'GON4L',\n",
       " 'SYT11': 'SYT11',\n",
       " 'RIT1': 'RIT1',\n",
       " 'KHDC4': 'KHDC4',\n",
       " 'ARHGEF2': 'ARHGEF2',\n",
       " 'SSR2': 'SSR2',\n",
       " 'UBQLN4': 'UBQLN4',\n",
       " 'LAMTOR2': 'LAMTOR2',\n",
       " 'LMNA': 'LMNA',\n",
       " 'SLC25A44': 'SLC25A44',\n",
       " 'PMF1': 'PMF1',\n",
       " 'SMG5': 'SMG5',\n",
       " 'GLMP': 'GLMP',\n",
       " 'CCT3': 'CCT3',\n",
       " 'MEF2D': 'MEF2D',\n",
       " 'IQGAP3': 'IQGAP3',\n",
       " 'NAXE': 'NAXE',\n",
       " 'GPATCH4': 'GPATCH4',\n",
       " 'ISG20L2': 'ISG20L2',\n",
       " 'MRPL24': 'MRPL24',\n",
       " 'HDGF': 'HDGF',\n",
       " 'PRCC': 'PRCC',\n",
       " 'KIRREL1': 'KIRREL1',\n",
       " 'IFI16': 'IFI16',\n",
       " 'DUSP23': 'DUSP23',\n",
       " 'TAGLN2': 'TAGLN2',\n",
       " 'PIGM': 'PIGM',\n",
       " 'PEA15': 'PEA15',\n",
       " 'DCAF8': 'DCAF8',\n",
       " 'PEX19': 'PEX19',\n",
       " 'COPA': 'COPA',\n",
       " 'NCSTN': 'NCSTN',\n",
       " 'PFDN2': 'PFDN2',\n",
       " 'NIT1': 'NIT1',\n",
       " 'DEDD': 'DEDD',\n",
       " 'UFC1': 'UFC1',\n",
       " 'B4GALT3': 'B4GALT3',\n",
       " 'NDUFS2': 'NDUFS2',\n",
       " 'SDHC': 'SDHC',\n",
       " 'DUSP12': 'DUSP12',\n",
       " 'ATF6': 'ATF6',\n",
       " 'UHMK1': 'UHMK1',\n",
       " 'UAP1': 'UAP1',\n",
       " 'DDR2': 'DDR2',\n",
       " 'RGS4': 'RGS4',\n",
       " 'NUF2': 'NUF2',\n",
       " 'PBX1': 'PBX1',\n",
       " 'MGST3': 'MGST3',\n",
       " 'ALDH9A1': 'ALDH9A1',\n",
       " 'TMCO1': 'TMCO1',\n",
       " 'UCK2': 'UCK2',\n",
       " 'POGK': 'POGK',\n",
       " 'POU2F1': 'POU2F1',\n",
       " 'MPZL1': 'MPZL1',\n",
       " 'MPC2': 'MPC2',\n",
       " 'DCAF6': 'DCAF6',\n",
       " 'GPR161': 'GPR161',\n",
       " 'TIPRL': 'TIPRL',\n",
       " 'SFT2D2': 'SFT2D2',\n",
       " 'ATP1B1': 'ATP1B1',\n",
       " 'NME7': 'NME7',\n",
       " 'BLZF1': 'BLZF1',\n",
       " 'KIFAP3': 'KIFAP3',\n",
       " 'GORAB': 'GORAB',\n",
       " 'PRRX1': 'PRRX1',\n",
       " 'PRRC2C': 'PRRC2C',\n",
       " 'VAMP4': 'VAMP4',\n",
       " 'EEF1AKNMT': 'EEF1AKNMT',\n",
       " 'PIGC': 'PIGC',\n",
       " 'SUCO': 'SUCO',\n",
       " 'PRDX6': 'PRDX6',\n",
       " 'KLHL20': 'KLHL20',\n",
       " 'CENPL': 'CENPL',\n",
       " 'DARS2': 'DARS2',\n",
       " 'GAS5': 'GAS5',\n",
       " 'ZBTB37': 'ZBTB37',\n",
       " 'RC3H1': 'RC3H1',\n",
       " 'RABGAP1L': 'RABGAP1L',\n",
       " 'CACYBP': 'CACYBP',\n",
       " 'MRPS14': 'MRPS14',\n",
       " 'COP1': 'COP1',\n",
       " 'RASAL2': 'RASAL2',\n",
       " 'RALGPS2': 'RALGPS2',\n",
       " 'FAM20B': 'FAM20B',\n",
       " 'TOR3A': 'TOR3A',\n",
       " 'ABL2': 'ABL2',\n",
       " 'SOAT1': 'SOAT1',\n",
       " 'TOR1AIP2': 'TOR1AIP2',\n",
       " 'TOR1AIP1': 'TOR1AIP1',\n",
       " 'CEP350': 'CEP350',\n",
       " 'QSOX1': 'QSOX1',\n",
       " 'ACBD6': 'ACBD6',\n",
       " 'XPR1': 'XPR1',\n",
       " 'STX6': 'STX6',\n",
       " 'MR1': 'MR1',\n",
       " 'IER5': 'IER5',\n",
       " 'GLUL': 'GLUL',\n",
       " 'DHX9': 'DHX9',\n",
       " 'LAMC1': 'LAMC1',\n",
       " 'NMNAT2': 'NMNAT2',\n",
       " 'SMG7': 'SMG7',\n",
       " 'ARPC5': 'ARPC5',\n",
       " 'RGL1': 'RGL1',\n",
       " 'TSEN15': 'TSEN15',\n",
       " 'C1orf21': 'C1orf21',\n",
       " 'EDEM3': 'EDEM3',\n",
       " 'NIBAN1': 'NIBAN1',\n",
       " 'RNF2': 'RNF2',\n",
       " 'TRMT1L': 'TRMT1L',\n",
       " 'IVNS1ABP': 'IVNS1ABP',\n",
       " 'TPR': 'TPR',\n",
       " 'ODR4': 'ODR4',\n",
       " 'PLA2G4A': 'PLA2G4A',\n",
       " 'UCHL5': 'UCHL5',\n",
       " 'RO60': 'RO60',\n",
       " 'GLRX2': 'GLRX2',\n",
       " 'CDC73': 'CDC73',\n",
       " 'CFH': 'CFH',\n",
       " 'ASPM': 'ASPM',\n",
       " 'ZBTB41': 'ZBTB41',\n",
       " 'DENND1B': 'DENND1B',\n",
       " 'C1orf53': 'C1orf53',\n",
       " 'NEK7': 'NEK7',\n",
       " 'MIR181A1HG': 'MIR181A1HG',\n",
       " 'ZNF281': 'ZNF281',\n",
       " 'KIF14': 'KIF14',\n",
       " 'DDX59': 'DDX59',\n",
       " 'CAMSAP2': 'CAMSAP2',\n",
       " 'TMEM9': 'TMEM9',\n",
       " 'PHLDA3': 'PHLDA3',\n",
       " 'CSRP1': 'CSRP1',\n",
       " 'NAV1': 'NAV1',\n",
       " 'IPO9': 'IPO9',\n",
       " 'SHISA4': 'SHISA4',\n",
       " 'TIMM17A': 'TIMM17A',\n",
       " 'RNPEP': 'RNPEP',\n",
       " 'ARL8A': 'ARL8A',\n",
       " 'UBE2T': 'UBE2T',\n",
       " 'KDM5B': 'KDM5B',\n",
       " 'RABIF': 'RABIF',\n",
       " 'ADIPOR1': 'ADIPOR1',\n",
       " 'CYB5R1': 'CYB5R1',\n",
       " 'TMEM183A': 'TMEM183A',\n",
       " 'ATP2B4': 'ATP2B4',\n",
       " 'SNRPE': 'SNRPE',\n",
       " 'PPP1R15B': 'PPP1R15B',\n",
       " 'MDM4': 'MDM4',\n",
       " 'NFASC': 'NFASC',\n",
       " 'RBBP5': 'RBBP5',\n",
       " 'DSTYK': 'DSTYK',\n",
       " 'ELK4': 'ELK4',\n",
       " 'NUCKS1': 'NUCKS1',\n",
       " 'RAB29': 'RAB29',\n",
       " 'SLC41A1': 'SLC41A1',\n",
       " 'SRGAP2': 'SRGAP2',\n",
       " 'EIF2D': 'EIF2D',\n",
       " 'DYRK3': 'DYRK3',\n",
       " 'MAPKAPK2': 'MAPKAPK2',\n",
       " 'CD55': 'CD55',\n",
       " 'CD46': 'CD46',\n",
       " 'PLXNA2': 'PLXNA2',\n",
       " 'G0S2': 'G0S2',\n",
       " 'UTP25': 'UTP25',\n",
       " 'SYT14': 'SYT14',\n",
       " 'SERTAD4-AS1': 'SERTAD4-AS1',\n",
       " 'RCOR3': 'RCOR3',\n",
       " 'TRAF5': 'TRAF5',\n",
       " 'LINC00467': 'LINC00467',\n",
       " 'SLC30A1': 'SLC30A1',\n",
       " 'NEK2': 'NEK2',\n",
       " 'LPGAT1': 'LPGAT1',\n",
       " 'INTS7': 'INTS7',\n",
       " 'DTL': 'DTL',\n",
       " 'PPP2R5A': 'PPP2R5A',\n",
       " 'PACC1': 'PACC1',\n",
       " 'NENF': 'NENF',\n",
       " 'ATF3': 'ATF3',\n",
       " 'NSL1': 'NSL1',\n",
       " 'TATDN3': 'TATDN3',\n",
       " 'ANGEL2': 'ANGEL2',\n",
       " 'RPS6KC1': 'RPS6KC1',\n",
       " 'SMYD2': 'SMYD2',\n",
       " 'PTPN14': 'PTPN14',\n",
       " 'CENPF': 'CENPF',\n",
       " 'KCNK2': 'KCNK2',\n",
       " 'KCTD3': 'KCTD3',\n",
       " 'GPATCH2': 'GPATCH2',\n",
       " 'RRP15': 'RRP15',\n",
       " 'TGFB2': 'TGFB2',\n",
       " 'LYPLAL1': 'LYPLAL1',\n",
       " 'EPRS': 'EPRS',\n",
       " 'BPNT1': 'BPNT1',\n",
       " 'IARS2': 'IARS2',\n",
       " 'RAB3GAP2': 'RAB3GAP2',\n",
       " 'MIA3': 'MIA3',\n",
       " 'AIDA': 'AIDA',\n",
       " 'BROX': 'BROX',\n",
       " 'AL392172.1': 'AL392172.1',\n",
       " 'CAPN2': 'CAPN2',\n",
       " 'TP53BP2': 'TP53BP2',\n",
       " 'FBXO28': 'FBXO28',\n",
       " 'DEGS1': 'DEGS1',\n",
       " 'NVL': 'NVL',\n",
       " 'CNIH4': 'CNIH4',\n",
       " 'WDR26': 'WDR26',\n",
       " 'DNAH14': 'DNAH14',\n",
       " 'LBR': 'LBR',\n",
       " 'ENAH': 'ENAH',\n",
       " 'SRP9': 'SRP9',\n",
       " 'EPHX1': 'EPHX1',\n",
       " 'TMEM63A': 'TMEM63A',\n",
       " 'PYCR2': 'PYCR2',\n",
       " 'SDE2': 'SDE2',\n",
       " 'H3F3A': 'H3F3A',\n",
       " 'ACBD3': 'ACBD3',\n",
       " 'LIN9': 'LIN9',\n",
       " 'PARP1': 'PARP1',\n",
       " 'PSEN2': 'PSEN2',\n",
       " 'CDC42BPA': 'CDC42BPA',\n",
       " 'SNAP47': 'SNAP47',\n",
       " 'JMJD4': 'JMJD4',\n",
       " 'ARF1': 'ARF1',\n",
       " 'C1orf35': 'C1orf35',\n",
       " 'MRPL55': 'MRPL55',\n",
       " 'GUK1': 'GUK1',\n",
       " 'TRIM11': 'TRIM11',\n",
       " 'HIST3H2A': 'HIST3H2A',\n",
       " 'RNF187': 'RNF187',\n",
       " 'RAB4A': 'RAB4A',\n",
       " 'CCSAP': 'CCSAP',\n",
       " 'NUP133': 'NUP133',\n",
       " 'ABCB10': 'ABCB10',\n",
       " 'TAF5L': 'TAF5L',\n",
       " 'GALNT2': 'GALNT2',\n",
       " 'COG2': 'COG2',\n",
       " 'C1orf198': 'C1orf198',\n",
       " 'ARV1': 'ARV1',\n",
       " 'C1orf131': 'C1orf131',\n",
       " 'GNPAT': 'GNPAT',\n",
       " 'EXOC8': 'EXOC8',\n",
       " 'SPRTN': 'SPRTN',\n",
       " 'EGLN1': 'EGLN1',\n",
       " 'AL445524.1': 'AL445524.1',\n",
       " 'TSNAX': 'TSNAX',\n",
       " 'NTPCR': 'NTPCR',\n",
       " 'COA6': 'COA6',\n",
       " 'TARBP1': 'TARBP1',\n",
       " 'IRF2BP2': 'IRF2BP2',\n",
       " 'TOMM20': 'TOMM20',\n",
       " 'ARID4B': 'ARID4B',\n",
       " 'GGPS1': 'GGPS1',\n",
       " 'B3GALNT2': 'B3GALNT2',\n",
       " 'LYST': 'LYST',\n",
       " 'NID1': 'NID1',\n",
       " 'LGALS8': 'LGALS8',\n",
       " 'HEATR1': 'HEATR1',\n",
       " 'MTR': 'MTR',\n",
       " 'FMN2': 'FMN2',\n",
       " 'FH': 'FH',\n",
       " 'OPN3': 'OPN3',\n",
       " 'CHML': 'CHML',\n",
       " 'EXO1': 'EXO1',\n",
       " 'CEP170': 'CEP170',\n",
       " 'SDCCAG8': 'SDCCAG8',\n",
       " 'AKT3': 'AKT3',\n",
       " 'ADSS': 'ADSS',\n",
       " 'DESI2': 'DESI2',\n",
       " 'COX20': 'COX20',\n",
       " 'HNRNPU': 'HNRNPU',\n",
       " 'EFCAB2': 'EFCAB2',\n",
       " 'SMYD3': 'SMYD3',\n",
       " 'TFB2M': 'TFB2M',\n",
       " 'CNST': 'CNST',\n",
       " 'SCCPDH': 'SCCPDH',\n",
       " 'AHCTF1': 'AHCTF1',\n",
       " 'ZNF496': 'ZNF496',\n",
       " 'SH3BP5L': 'SH3BP5L',\n",
       " 'ZNF672': 'ZNF672',\n",
       " 'SH3YL1': 'SH3YL1',\n",
       " 'ACP1': 'ACP1',\n",
       " 'TMEM18': 'TMEM18',\n",
       " 'PXDN': 'PXDN',\n",
       " 'EIPR1': 'EIPR1',\n",
       " 'TRAPPC12': 'TRAPPC12',\n",
       " 'ADI1': 'ADI1',\n",
       " 'RNASEH1': 'RNASEH1',\n",
       " 'RNASEH1-AS1': 'RNASEH1-AS1',\n",
       " 'RPS7': 'RPS7',\n",
       " 'SOX11': 'SOX11',\n",
       " 'ID2': 'ID2',\n",
       " 'KIDINS220': 'KIDINS220',\n",
       " 'MBOAT2': 'MBOAT2',\n",
       " 'ASAP2': 'ASAP2',\n",
       " 'ITGB1BP1': 'ITGB1BP1',\n",
       " 'CPSF3': 'CPSF3',\n",
       " 'IAH1': 'IAH1',\n",
       " 'ADAM17': 'ADAM17',\n",
       " 'YWHAQ': 'YWHAQ',\n",
       " 'TAF1B': 'TAF1B',\n",
       " 'RRM2': 'RRM2',\n",
       " 'HPCAL1': 'HPCAL1',\n",
       " 'ODC1': 'ODC1',\n",
       " 'NOL10': 'NOL10',\n",
       " 'PDIA6': 'PDIA6',\n",
       " 'ROCK2': 'ROCK2',\n",
       " 'E2F6': 'E2F6',\n",
       " 'LPIN1': 'LPIN1',\n",
       " 'NBAS': 'NBAS',\n",
       " 'DDX1': 'DDX1',\n",
       " 'SMC6': 'SMC6',\n",
       " 'GEN1': 'GEN1',\n",
       " 'RDH14': 'RDH14',\n",
       " 'TTC32': 'TTC32',\n",
       " 'WDR35': 'WDR35',\n",
       " 'LAPTM4A': 'LAPTM4A',\n",
       " 'SDC1': 'SDC1',\n",
       " 'PUM2': 'PUM2',\n",
       " 'RHOB': 'RHOB',\n",
       " 'HS1BP3': 'HS1BP3',\n",
       " 'LDAH': 'LDAH',\n",
       " 'UBXN2A': 'UBXN2A',\n",
       " 'FKBP1B': 'FKBP1B',\n",
       " 'SF3B6': 'SF3B6',\n",
       " 'FAM228B': 'FAM228B',\n",
       " 'TP53I3': 'TP53I3',\n",
       " 'ITSN2': 'ITSN2',\n",
       " 'NCOA1': 'NCOA1',\n",
       " 'PTRHD1': 'PTRHD1',\n",
       " 'CENPO': 'CENPO',\n",
       " 'ADCY3': 'ADCY3',\n",
       " 'ASXL2': 'ASXL2',\n",
       " 'KIF3C': 'KIF3C',\n",
       " 'RAB10': 'RAB10',\n",
       " 'HADHA': 'HADHA',\n",
       " 'HADHB': 'HADHB',\n",
       " 'SELENOI': 'SELENOI',\n",
       " 'SLC35F6': 'SLC35F6',\n",
       " 'CENPA': 'CENPA',\n",
       " 'MAPRE3': 'MAPRE3',\n",
       " 'TMEM214': 'TMEM214',\n",
       " 'AGBL5': 'AGBL5',\n",
       " 'OST4': 'OST4',\n",
       " 'PREB': 'PREB',\n",
       " 'SLC5A6': 'SLC5A6',\n",
       " 'ATRAID': 'ATRAID',\n",
       " 'CAD': 'CAD',\n",
       " 'MPV17': 'MPV17',\n",
       " 'GTF3C2': 'GTF3C2',\n",
       " 'EIF2B4': 'EIF2B4',\n",
       " 'SNX17': 'SNX17',\n",
       " 'PPM1G': 'PPM1G',\n",
       " 'NRBP1': 'NRBP1',\n",
       " 'FNDC4': 'FNDC4',\n",
       " 'ZNF512': 'ZNF512',\n",
       " 'GPN1': 'GPN1',\n",
       " 'SUPT7L': 'SUPT7L',\n",
       " 'SLC4A1AP': 'SLC4A1AP',\n",
       " 'MRPL33': 'MRPL33',\n",
       " 'BABAM2': 'BABAM2',\n",
       " 'FOSL2': 'FOSL2',\n",
       " 'PPP1CB': 'PPP1CB',\n",
       " 'TRMT61B': 'TRMT61B',\n",
       " 'WDR43': 'WDR43',\n",
       " 'CLIP4': 'CLIP4',\n",
       " 'YPEL5': 'YPEL5',\n",
       " 'LBH': 'LBH',\n",
       " 'LCLAT1': 'LCLAT1',\n",
       " 'EHD3': 'EHD3',\n",
       " 'DPY30': 'DPY30',\n",
       " 'SPAST': 'SPAST',\n",
       " 'SLC30A6': 'SLC30A6',\n",
       " 'YIPF4': 'YIPF4',\n",
       " 'BIRC6': 'BIRC6',\n",
       " 'TTC27': 'TTC27',\n",
       " 'LTBP1': 'LTBP1',\n",
       " 'FAM98A': 'FAM98A',\n",
       " 'CRIM1-DT': 'CRIM1-DT',\n",
       " 'CRIM1': 'CRIM1',\n",
       " 'FEZ2': 'FEZ2',\n",
       " 'STRN': 'STRN',\n",
       " 'GPATCH11': 'GPATCH11',\n",
       " 'EIF2AK2': 'EIF2AK2',\n",
       " 'CEBPZOS': 'CEBPZOS',\n",
       " 'CEBPZ': 'CEBPZ',\n",
       " 'NDUFAF7': 'NDUFAF7',\n",
       " 'PRKD3': 'PRKD3',\n",
       " 'CDC42EP3': 'CDC42EP3',\n",
       " 'CYP1B1': 'CYP1B1',\n",
       " 'ATL2': 'ATL2',\n",
       " 'HNRNPLL': 'HNRNPLL',\n",
       " 'SRSF7': 'SRSF7',\n",
       " 'GEMIN6': 'GEMIN6',\n",
       " 'DHX57': 'DHX57',\n",
       " 'MORN2': 'MORN2',\n",
       " 'SOS1': 'SOS1',\n",
       " 'MAP4K3': 'MAP4K3',\n",
       " 'MAP4K3-DT': 'MAP4K3-DT',\n",
       " 'THUMPD2': 'THUMPD2',\n",
       " 'SLC8A1': 'SLC8A1',\n",
       " 'EML4': 'EML4',\n",
       " 'COX7A2L': 'COX7A2L',\n",
       " 'MTA3': 'MTA3',\n",
       " 'ZFP36L2': 'ZFP36L2',\n",
       " ...}"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "map_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "3e25af68",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>You are an expert in single-cell biology and f...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>You are a domain expert in functional genomics...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>You are assisting with the interpretation of p...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>You are an expert in regulatory genomics. Cons...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>You are a specialist in gene perturbation expe...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62065</th>\n",
       "      <td>You are an expert in gene expression analysis....</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62066</th>\n",
       "      <td>You are a genomics expert evaluating perturbat...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62067</th>\n",
       "      <td>You are an expert analyst of perturbation data...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62068</th>\n",
       "      <td>You are a functional genomics specialist. In K...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62069</th>\n",
       "      <td>You are an expert in gene regulation studies. ...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>62070 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                question answer\n",
       "0      You are an expert in single-cell biology and f...    Yes\n",
       "1      You are a domain expert in functional genomics...    Yes\n",
       "2      You are assisting with the interpretation of p...    Yes\n",
       "3      You are an expert in regulatory genomics. Cons...    Yes\n",
       "4      You are a specialist in gene perturbation expe...    Yes\n",
       "...                                                  ...    ...\n",
       "62065  You are an expert in gene expression analysis....    Yes\n",
       "62066  You are a genomics expert evaluating perturbat...    Yes\n",
       "62067  You are an expert analyst of perturbation data...    Yes\n",
       "62068  You are a functional genomics specialist. In K...    Yes\n",
       "62069  You are an expert in gene regulation studies. ...    Yes\n",
       "\n",
       "[62070 rows x 2 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "1b06f88e",
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "replace() argument 2 must be str, not Series",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mTypeError\u001b[39m                                 Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[38]\u001b[39m\u001b[32m, line 5\u001b[39m\n\u001b[32m      3\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m j \u001b[38;5;129;01min\u001b[39;00m map_dict.keys():\n\u001b[32m      4\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m j \u001b[38;5;129;01min\u001b[39;00m item:\n\u001b[32m----> \u001b[39m\u001b[32m5\u001b[39m         item_new = item.replace(j, map_dict[j])\n\u001b[32m      6\u001b[39m         que_new.append(item_new)\n",
      "\u001b[31mTypeError\u001b[39m: replace() argument 2 must be str, not Series"
     ]
    }
   ],
   "source": [
    "que_new = []\n",
    "for item in df['question'].values:\n",
    "    for j in map_dict.keys():\n",
    "        if j in item:\n",
    "            item_new = item.replace(j, map_dict[j])\n",
    "            que_new.append(item_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f00abb40",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(que_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "bdbfd195",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['question_new'] = df['question'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "abc21ac2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.model_selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "8e3be03a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>question_new</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>You are an expert in single-cell biology and f...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are an expert in single-cell biology and f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>You are a domain expert in functional genomics...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are a domain expert in functional genomics...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>You are assisting with the interpretation of p...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are assisting with the interpretation of p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>You are an expert in regulatory genomics. Cons...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are an expert in regulatory genomics. Cons...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>You are a specialist in gene perturbation expe...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are a specialist in gene perturbation expe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62065</th>\n",
       "      <td>You are an expert in gene expression analysis....</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are an expert in gene expression analysis....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62066</th>\n",
       "      <td>You are a genomics expert evaluating perturbat...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are a genomics expert evaluating perturbat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62067</th>\n",
       "      <td>You are an expert analyst of perturbation data...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are an expert analyst of perturbation data...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62068</th>\n",
       "      <td>You are a functional genomics specialist. In K...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are a functional genomics specialist. In K...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62069</th>\n",
       "      <td>You are an expert in gene regulation studies. ...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>You are an expert in gene regulation studies. ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>62070 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                question answer  \\\n",
       "0      You are an expert in single-cell biology and f...    Yes   \n",
       "1      You are a domain expert in functional genomics...    Yes   \n",
       "2      You are assisting with the interpretation of p...    Yes   \n",
       "3      You are an expert in regulatory genomics. Cons...    Yes   \n",
       "4      You are a specialist in gene perturbation expe...    Yes   \n",
       "...                                                  ...    ...   \n",
       "62065  You are an expert in gene expression analysis....    Yes   \n",
       "62066  You are a genomics expert evaluating perturbat...    Yes   \n",
       "62067  You are an expert analyst of perturbation data...    Yes   \n",
       "62068  You are a functional genomics specialist. In K...    Yes   \n",
       "62069  You are an expert in gene regulation studies. ...    Yes   \n",
       "\n",
       "                                            question_new  \n",
       "0      You are an expert in single-cell biology and f...  \n",
       "1      You are a domain expert in functional genomics...  \n",
       "2      You are assisting with the interpretation of p...  \n",
       "3      You are an expert in regulatory genomics. Cons...  \n",
       "4      You are a specialist in gene perturbation expe...  \n",
       "...                                                  ...  \n",
       "62065  You are an expert in gene expression analysis....  \n",
       "62066  You are a genomics expert evaluating perturbat...  \n",
       "62067  You are an expert analyst of perturbation data...  \n",
       "62068  You are a functional genomics specialist. In K...  \n",
       "62069  You are an expert in gene regulation studies. ...  \n",
       "\n",
       "[62070 rows x 3 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "fc92dc06",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_id, test_id = sklearn.model_selection.train_test_split(list(df.index), test_size=3000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "c3c35484",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "answer\n",
       "Yes    49049\n",
       "No     10021\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[train_id]['answer'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "34eb054c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[train_id].to_csv(\"reploge_train.csv\")\n",
    "df.loc[test_id].to_csv(\"reploge_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "f40b6710",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3000"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df.loc[test_id])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "81f6a294",
   "metadata": {},
   "outputs": [],
   "source": [
    "# /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/adamson_train.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "e8cecc33",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_d = df.loc[train_id]\n",
    "train_d['_id'] = list(train_d.index)\n",
    "\n",
    "test_d = df.loc[test_id]\n",
    "test_d['_id'] = list(test_d.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "36014a2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_d.to_json(\"./reploge_train.json\", orient='records', indent=2)\n",
    "test_d.to_json(\"./reploge_test.json\", orient='records', indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fcb6a5ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45a5648c",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.read_csv(\"./\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0029a293",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "55c6e1b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import sklearn.model_selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "8821fa32",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"/gpfs/radev/project/zhao/hs2234/evox/QA/gene_perturb/qa/adamson_psa_single_qa.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "68c21121",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['question_new'] = df['question']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "4b87a841",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_id, test_id = sklearn.model_selection.train_test_split(list(df.index), test_size=3000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "6603c2a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[train_id].to_csv(\"adamson_update_train.csv\")\n",
    "df.loc[test_id].to_csv(\"adamson_update_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "329b1b04",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_d = df.loc[train_id]\n",
    "train_d['_id'] = list(train_d.index)\n",
    "\n",
    "test_d = df.loc[test_id]\n",
    "test_d['_id'] = list(test_d.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "44a7904f",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_d.to_json(\"./adamson_update_train.json\", orient='records', indent=2)\n",
    "test_d.to_json(\"./adamson_update_test.json\", orient='records', indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "418cb72f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "58428a70",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import sklearn.model_selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7f4f7c39",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"/gpfs/radev/project/zhao/hs2234/evox/QA/gene_perturb/qa/replogle_k562_psa_single_qa.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e2e2c3f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['question_new'] = df['question']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "847895b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_id, test_id = sklearn.model_selection.train_test_split(list(df.index), test_size=3000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "26f72b44",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[train_id].to_csv(\"replogle_update_train.csv\")\n",
    "df.loc[test_id].to_csv(\"replogle_update_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f29d9f21",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_d = df.loc[train_id]\n",
    "train_d['_id'] = list(train_d.index)\n",
    "\n",
    "test_d = df.loc[test_id]\n",
    "test_d['_id'] = list(test_d.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6153dca3",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_d.to_json(\"./replogle_update_train.json\", orient='records', indent=2)\n",
    "test_d.to_json(\"./replogle_update_test.json\", orient='records', indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f4b5c74",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}