{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d83f232a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7f3a1697",
   "metadata": {},
   "outputs": [],
   "source": [
    "%pwd\n",
    "os.chdir(\"../\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "70433f0a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'c:\\\\order\\\\Desktop\\\\AI_Summarizer'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4b141e83",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dataclasses import dataclass\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ffb2ded1",
   "metadata": {},
   "outputs": [],
   "source": [
    "@dataclass(frozen=True)\n",
    "class ModelEvaluationConfig:\n",
    "    root_dir: Path\n",
    "    data_path: Path\n",
    "    model_path: Path\n",
    "    tokenizer_path: Path\n",
    "    metric_file_name: Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4cbe6e7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from textSummarizer.constants import *\n",
    "from textSummarizer.utils.common import read_yaml, create_directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6ef66517",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConfigurationManager:\n",
    "    def __init__(\n",
    "        self,\n",
    "        config_filepath = CONFIG_FILE_PATH,\n",
    "        params_filepath = PARAMS_FILE_PATH):\n",
    "        \n",
    "        self.config = read_yaml(config_filepath)\n",
    "        self.params = read_yaml(params_filepath)\n",
    "        \n",
    "        create_directories([self.config.artifacts_root])\n",
    "        \n",
    "        \n",
    "    def  get_model_evaluation_config(self) -> ModelEvaluationConfig:\n",
    "        config = self.config.model_evaluation\n",
    "        \n",
    "        create_directories([config.root_dir])\n",
    "        \n",
    "        model_evaluation_config = ModelEvaluationConfig(\n",
    "            root_dir = config.root_dir,\n",
    "            data_path= config.data_path,\n",
    "            model_path = config.model_path,\n",
    "            tokenizer_path= config.tokenizer_path,\n",
    "            metric_file_name = config.metric_file_name\n",
    "        )\n",
    "        \n",
    "        return model_evaluation_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "98b6adaf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\jeeva\\.conda\\envs\\textSummarizer\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2025-07-02 03:44:43,649: INFO: config : PyTorch version 2.5.1+cu121 available. ]\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n",
    "from datasets import load_from_disk, load_dataset\n",
    "import evaluate\n",
    "import torch\n",
    "import pandas as pd\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ac50b978",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ModelEvaluation:\n",
    "    def __init__(self, config: ModelEvaluationConfig):\n",
    "        self.config = config\n",
    "    \n",
    "    \n",
    "    def generate_batch_sized_chunks(self, List_of_elements, batch_size):\n",
    "        \"\"\"split the dataset into smaller batches that we can process simulaneously\n",
    "        yield successive batch-sized chunks from list_of_elements.\"\"\"\n",
    "        \n",
    "        for i in range(0, len(List_of_elements), batch_size):\n",
    "            yield List_of_elements[i : i + batch_size]\n",
    "            \n",
    "    def calculate_metric_on_test_ds(self, dataset, metric, model, tokenizer,\n",
    "                                    batch_size=16, device=\"cuda\" if torch.cuda.is_available() else \"cpu\",\n",
    "                                    column_text=\"article\",                \n",
    "                                    column_summary=\"highlights\"):\n",
    "        article_batches = list(self.generate_batch_sized_chunks(dataset[column_text], batch_size))\n",
    "        target_batches = list(self.generate_batch_sized_chunks(dataset[column_summary], batch_size))\n",
    "        \n",
    "        for article_batch, target_batch in tqdm(\n",
    "            zip(article_batches, target_batches), total=len(article_batches)):\n",
    "            \n",
    "            inputs = tokenizer( article_batch, max_length=1024, truncation=True,\n",
    "                padding=\"max_length\", return_tensors=\"pt\")\n",
    "            \n",
    "            summaries = model.generate(inputs[\"input_ids\"].todevice(device),\n",
    "                                       attention_mask=inputs[\"attention_mask\"].todevice(device),\n",
    "                                       Length_penalty=0.8, num_beams=8, max_length=128)\n",
    "            ''' parameter for length penalty ensures that model does not generate sequences that are too long '''\n",
    "        # Finally, we decode the generated texts,\n",
    "        # replace the token and add the decoded texts with the references to the metrics\n",
    "        \n",
    "            decoded_summaries = [tokenizer.batch_decode(s, skip_special_tokens=True,\n",
    "                                    clean_up_tokenization_spaces= True) \n",
    "                for s in summaries]\n",
    "            \n",
    "            decoded_summaries = [d.replace(\"\", \" \") for d in decoded_summaries]\n",
    "            \n",
    "            metric.add_batch(predictions=decoded_summaries, references=target_batch)\n",
    "            \n",
    "        score = metric.compute()\n",
    "        return score           \n",
    "            \n",
    "    def evaluate(self):\n",
    "        device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "        tokenizer = AutoTokenizer.from_pretrained(self.config.tokenizer_path)\n",
    "        model_pegasus = AutoModelForSeq2SeqLM.from_pretrained(self.config.model_path).to(device)\n",
    "        \n",
    "        dataset_samsum_pt = load_from_disk(self.config.data_path)\n",
    "        \n",
    "        rouge_names = [\"rouge1\", \"rouge2\", \"rougeL\", \"rougeLsum\"]\n",
    "        \n",
    "        rouge_metric = evaluate.load(\"rouge\")\n",
    "        \n",
    "        score = self.calculate_metric_on_test_ds(\n",
    "            dataset_samsum_pt['test'][0:10], rouge_metric, model_pegasus, tokenizer,  batch_size = 2, column_text = 'dialogue', column_summary= 'summary')\n",
    "        \n",
    "        rouge_dict = dict((rn, score[rn].mid.fmeasure ) for rn in rouge_names )\n",
    "\n",
    "        df = pd.DataFrame(rouge_dict, index = ['pegasus'] )\n",
    "        df.to_csv(self.config.metric_file_name, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "54baa53e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2025-07-02 03:44:49,371: INFO: common : yaml. file: config\\config.yaml loaded successfully. ]\n",
      "[2025-07-02 03:44:49,375: INFO: common : yaml. file: params.yaml loaded successfully. ]\n",
      "[2025-07-02 03:44:49,375: INFO: common : created directory at path : artifacts_root ]\n",
      "[2025-07-02 03:44:49,375: INFO: common : created directory at path : artifacts/model_evaluation ]\n"
     ]
    },
    {
     "ename": "HFValidationError",
     "evalue": "Repo id must be in the form 'repo_name' or 'namespace/repo_name': 'artifacts/model_trainer/tokenizer_name'. Use `repo_type` argument if needed.",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mHFValidationError\u001b[39m                         Traceback (most recent call last)",
      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jeeva\\.conda\\envs\\textSummarizer\\Lib\\site-packages\\transformers\\utils\\hub.py:470\u001b[39m, in \u001b[36mcached_files\u001b[39m\u001b[34m(path_or_repo_id, filenames, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, subfolder, repo_type, user_agent, _raise_exceptions_for_gated_repo, _raise_exceptions_for_missing_entries, _raise_exceptions_for_connection_errors, _commit_hash, **deprecated_kwargs)\u001b[39m\n\u001b[32m    468\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(full_filenames) == \u001b[32m1\u001b[39m:\n\u001b[32m    469\u001b[39m     \u001b[38;5;66;03m# This is slightly better for only 1 file\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m470\u001b[39m     \u001b[43mhf_hub_download\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    471\u001b[39m \u001b[43m        \u001b[49m\u001b[43mpath_or_repo_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    472\u001b[39m \u001b[43m        \u001b[49m\u001b[43mfilenames\u001b[49m\u001b[43m[\u001b[49m\u001b[32;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    473\u001b[39m \u001b[43m        \u001b[49m\u001b[43msubfolder\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43msubfolder\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[43m==\u001b[49m\u001b[43m \u001b[49m\u001b[32;43m0\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43msubfolder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    474\u001b[39m \u001b[43m        \u001b[49m\u001b[43mrepo_type\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrepo_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    475\u001b[39m \u001b[43m        \u001b[49m\u001b[43mrevision\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrevision\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    476\u001b[39m \u001b[43m        \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    477\u001b[39m \u001b[43m        \u001b[49m\u001b[43muser_agent\u001b[49m\u001b[43m=\u001b[49m\u001b[43muser_agent\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    478\u001b[39m \u001b[43m        \u001b[49m\u001b[43mforce_download\u001b[49m\u001b[43m=\u001b[49m\u001b[43mforce_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    479\u001b[39m \u001b[43m        \u001b[49m\u001b[43mproxies\u001b[49m\u001b[43m=\u001b[49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    480\u001b[39m \u001b[43m        \u001b[49m\u001b[43mresume_download\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresume_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    481\u001b[39m \u001b[43m        \u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    482\u001b[39m \u001b[43m        \u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    483\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    484\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jeeva\\.conda\\envs\\textSummarizer\\Lib\\site-packages\\huggingface_hub\\utils\\_validators.py:106\u001b[39m, in \u001b[36mvalidate_hf_hub_args.<locals>._inner_fn\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m    105\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m arg_name \u001b[38;5;129;01min\u001b[39;00m [\u001b[33m\"\u001b[39m\u001b[33mrepo_id\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mfrom_id\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mto_id\u001b[39m\u001b[33m\"\u001b[39m]:\n\u001b[32m--> \u001b[39m\u001b[32m106\u001b[39m     \u001b[43mvalidate_repo_id\u001b[49m\u001b[43m(\u001b[49m\u001b[43marg_value\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    108\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m arg_name == \u001b[33m\"\u001b[39m\u001b[33mtoken\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m arg_value \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jeeva\\.conda\\envs\\textSummarizer\\Lib\\site-packages\\huggingface_hub\\utils\\_validators.py:154\u001b[39m, in \u001b[36mvalidate_repo_id\u001b[39m\u001b[34m(repo_id)\u001b[39m\n\u001b[32m    153\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m repo_id.count(\u001b[33m\"\u001b[39m\u001b[33m/\u001b[39m\u001b[33m\"\u001b[39m) > \u001b[32m1\u001b[39m:\n\u001b[32m--> \u001b[39m\u001b[32m154\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m HFValidationError(\n\u001b[32m    155\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mRepo id must be in the form \u001b[39m\u001b[33m'\u001b[39m\u001b[33mrepo_name\u001b[39m\u001b[33m'\u001b[39m\u001b[33m or \u001b[39m\u001b[33m'\u001b[39m\u001b[33mnamespace/repo_name\u001b[39m\u001b[33m'\u001b[39m\u001b[33m:\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    156\u001b[39m         \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mrepo_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m. Use `repo_type` argument if needed.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    157\u001b[39m     )\n\u001b[32m    159\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m REPO_ID_REGEX.match(repo_id):\n",
      "\u001b[31mHFValidationError\u001b[39m: Repo id must be in the form 'repo_name' or 'namespace/repo_name': 'artifacts/model_trainer/tokenizer_name'. Use `repo_type` argument if needed.",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[31mHFValidationError\u001b[39m                         Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[10]\u001b[39m\u001b[32m, line 7\u001b[39m\n\u001b[32m      5\u001b[39m     model_evaluation_config.evaluate()\n\u001b[32m      6\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m----> \u001b[39m\u001b[32m7\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m e\n",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[10]\u001b[39m\u001b[32m, line 5\u001b[39m\n\u001b[32m      3\u001b[39m     model_evaluation_config = config.get_model_evaluation_config()\n\u001b[32m      4\u001b[39m     model_evaluation_config = ModelEvaluation(config=model_evaluation_config)\n\u001b[32m----> \u001b[39m\u001b[32m5\u001b[39m     \u001b[43mmodel_evaluation_config\u001b[49m\u001b[43m.\u001b[49m\u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m      6\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m      7\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m e\n",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[9]\u001b[39m\u001b[32m, line 46\u001b[39m, in \u001b[36mModelEvaluation.evaluate\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m     44\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mevaluate\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[32m     45\u001b[39m     device = \u001b[33m\"\u001b[39m\u001b[33mcuda\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m torch.cuda.is_available() \u001b[38;5;28;01melse\u001b[39;00m \u001b[33m\"\u001b[39m\u001b[33mcpu\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m---> \u001b[39m\u001b[32m46\u001b[39m     tokenizer = \u001b[43mAutoTokenizer\u001b[49m\u001b[43m.\u001b[49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m.\u001b[49m\u001b[43mtokenizer_path\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     47\u001b[39m     model_pegasus = AutoModelForSeq2SeqLM.from_pretrained(\u001b[38;5;28mself\u001b[39m.config.model_path).to(device)\n\u001b[32m     49\u001b[39m     dataset_samsum_pt = load_from_disk(\u001b[38;5;28mself\u001b[39m.config.data_path)\n",
      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jeeva\\.conda\\envs\\textSummarizer\\Lib\\site-packages\\transformers\\models\\auto\\tokenization_auto.py:982\u001b[39m, in \u001b[36mAutoTokenizer.from_pretrained\u001b[39m\u001b[34m(cls, pretrained_model_name_or_path, *inputs, **kwargs)\u001b[39m\n\u001b[32m    979\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)\n\u001b[32m    981\u001b[39m \u001b[38;5;66;03m# Next, let's try to use the tokenizer_config file to get the tokenizer class.\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m982\u001b[39m tokenizer_config = \u001b[43mget_tokenizer_config\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpretrained_model_name_or_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    983\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[33m\"\u001b[39m\u001b[33m_commit_hash\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m tokenizer_config:\n\u001b[32m    984\u001b[39m     kwargs[\u001b[33m\"\u001b[39m\u001b[33m_commit_hash\u001b[39m\u001b[33m\"\u001b[39m] = tokenizer_config[\u001b[33m\"\u001b[39m\u001b[33m_commit_hash\u001b[39m\u001b[33m\"\u001b[39m]\n",
      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jeeva\\.conda\\envs\\textSummarizer\\Lib\\site-packages\\transformers\\models\\auto\\tokenization_auto.py:814\u001b[39m, in \u001b[36mget_tokenizer_config\u001b[39m\u001b[34m(pretrained_model_name_or_path, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, subfolder, **kwargs)\u001b[39m\n\u001b[32m    811\u001b[39m     token = use_auth_token\n\u001b[32m    813\u001b[39m commit_hash = kwargs.get(\u001b[33m\"\u001b[39m\u001b[33m_commit_hash\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[32m--> \u001b[39m\u001b[32m814\u001b[39m resolved_config_file = \u001b[43mcached_file\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    815\u001b[39m \u001b[43m    \u001b[49m\u001b[43mpretrained_model_name_or_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    816\u001b[39m \u001b[43m    \u001b[49m\u001b[43mTOKENIZER_CONFIG_FILE\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    817\u001b[39m \u001b[43m    \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    818\u001b[39m \u001b[43m    \u001b[49m\u001b[43mforce_download\u001b[49m\u001b[43m=\u001b[49m\u001b[43mforce_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    819\u001b[39m \u001b[43m    \u001b[49m\u001b[43mresume_download\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresume_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    820\u001b[39m \u001b[43m    \u001b[49m\u001b[43mproxies\u001b[49m\u001b[43m=\u001b[49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    821\u001b[39m \u001b[43m    \u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    822\u001b[39m \u001b[43m    \u001b[49m\u001b[43mrevision\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrevision\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    823\u001b[39m \u001b[43m    \u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    824\u001b[39m \u001b[43m    \u001b[49m\u001b[43msubfolder\u001b[49m\u001b[43m=\u001b[49m\u001b[43msubfolder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    825\u001b[39m \u001b[43m    \u001b[49m\u001b[43m_raise_exceptions_for_gated_repo\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m    826\u001b[39m \u001b[43m    \u001b[49m\u001b[43m_raise_exceptions_for_missing_entries\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m    827\u001b[39m \u001b[43m    \u001b[49m\u001b[43m_raise_exceptions_for_connection_errors\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m    828\u001b[39m \u001b[43m    \u001b[49m\u001b[43m_commit_hash\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcommit_hash\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    829\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    830\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m resolved_config_file \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m    831\u001b[39m     logger.info(\u001b[33m\"\u001b[39m\u001b[33mCould not locate the tokenizer configuration file, will try to use the model config instead.\u001b[39m\u001b[33m\"\u001b[39m)\n",
      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jeeva\\.conda\\envs\\textSummarizer\\Lib\\site-packages\\transformers\\utils\\hub.py:312\u001b[39m, in \u001b[36mcached_file\u001b[39m\u001b[34m(path_or_repo_id, filename, **kwargs)\u001b[39m\n\u001b[32m    254\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mcached_file\u001b[39m(\n\u001b[32m    255\u001b[39m     path_or_repo_id: Union[\u001b[38;5;28mstr\u001b[39m, os.PathLike],\n\u001b[32m    256\u001b[39m     filename: \u001b[38;5;28mstr\u001b[39m,\n\u001b[32m    257\u001b[39m     **kwargs,\n\u001b[32m    258\u001b[39m ) -> Optional[\u001b[38;5;28mstr\u001b[39m]:\n\u001b[32m    259\u001b[39m \u001b[38;5;250m    \u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m    260\u001b[39m \u001b[33;03m    Tries to locate a file in a local folder and repo, downloads and cache it if necessary.\u001b[39;00m\n\u001b[32m    261\u001b[39m \n\u001b[32m   (...)\u001b[39m\u001b[32m    310\u001b[39m \u001b[33;03m    ```\u001b[39;00m\n\u001b[32m    311\u001b[39m \u001b[33;03m    \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m312\u001b[39m     file = \u001b[43mcached_files\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath_or_repo_id\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpath_or_repo_id\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfilenames\u001b[49m\u001b[43m=\u001b[49m\u001b[43m[\u001b[49m\u001b[43mfilename\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    313\u001b[39m     file = file[\u001b[32m0\u001b[39m] \u001b[38;5;28;01mif\u001b[39;00m file \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m file\n\u001b[32m    314\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m file\n",
      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jeeva\\.conda\\envs\\textSummarizer\\Lib\\site-packages\\transformers\\utils\\hub.py:522\u001b[39m, in \u001b[36mcached_files\u001b[39m\u001b[34m(path_or_repo_id, filenames, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, subfolder, repo_type, user_agent, _raise_exceptions_for_gated_repo, _raise_exceptions_for_missing_entries, _raise_exceptions_for_connection_errors, _commit_hash, **deprecated_kwargs)\u001b[39m\n\u001b[32m    515\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m(\n\u001b[32m    516\u001b[39m         \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mPermissionError at \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me.filename\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m when downloading \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpath_or_repo_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m. \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    517\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mCheck cache directory permissions. Common causes: 1) another user is downloading the same model (please wait); \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    518\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33m2) a previous download was canceled and the lock file needs manual removal.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    519\u001b[39m     ) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01me\u001b[39;00m\n\u001b[32m    521\u001b[39m \u001b[38;5;66;03m# Now we try to recover if we can find all files correctly in the cache\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m522\u001b[39m resolved_files = \u001b[43m[\u001b[49m\n\u001b[32m    523\u001b[39m \u001b[43m    \u001b[49m\u001b[43m_get_cache_file_to_return\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath_or_repo_id\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfilename\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrevision\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mfilename\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mfull_filenames\u001b[49m\n\u001b[32m    524\u001b[39m \u001b[43m\u001b[49m\u001b[43m]\u001b[49m\n\u001b[32m    525\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mall\u001b[39m(file \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01mfor\u001b[39;00m file \u001b[38;5;129;01min\u001b[39;00m resolved_files):\n\u001b[32m    526\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m resolved_files\n",
      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jeeva\\.conda\\envs\\textSummarizer\\Lib\\site-packages\\transformers\\utils\\hub.py:523\u001b[39m, in \u001b[36m<listcomp>\u001b[39m\u001b[34m(.0)\u001b[39m\n\u001b[32m    515\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m(\n\u001b[32m    516\u001b[39m         \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mPermissionError at \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me.filename\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m when downloading \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpath_or_repo_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m. \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    517\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mCheck cache directory permissions. Common causes: 1) another user is downloading the same model (please wait); \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    518\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33m2) a previous download was canceled and the lock file needs manual removal.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    519\u001b[39m     ) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01me\u001b[39;00m\n\u001b[32m    521\u001b[39m \u001b[38;5;66;03m# Now we try to recover if we can find all files correctly in the cache\u001b[39;00m\n\u001b[32m    522\u001b[39m resolved_files = [\n\u001b[32m--> \u001b[39m\u001b[32m523\u001b[39m     \u001b[43m_get_cache_file_to_return\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath_or_repo_id\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfilename\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrevision\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m filename \u001b[38;5;129;01min\u001b[39;00m full_filenames\n\u001b[32m    524\u001b[39m ]\n\u001b[32m    525\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mall\u001b[39m(file \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01mfor\u001b[39;00m file \u001b[38;5;129;01min\u001b[39;00m resolved_files):\n\u001b[32m    526\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m resolved_files\n",
      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jeeva\\.conda\\envs\\textSummarizer\\Lib\\site-packages\\transformers\\utils\\hub.py:140\u001b[39m, in \u001b[36m_get_cache_file_to_return\u001b[39m\u001b[34m(path_or_repo_id, full_filename, cache_dir, revision)\u001b[39m\n\u001b[32m    136\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m_get_cache_file_to_return\u001b[39m(\n\u001b[32m    137\u001b[39m     path_or_repo_id: \u001b[38;5;28mstr\u001b[39m, full_filename: \u001b[38;5;28mstr\u001b[39m, cache_dir: Union[\u001b[38;5;28mstr\u001b[39m, Path, \u001b[38;5;28;01mNone\u001b[39;00m] = \u001b[38;5;28;01mNone\u001b[39;00m, revision: Optional[\u001b[38;5;28mstr\u001b[39m] = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m    138\u001b[39m ):\n\u001b[32m    139\u001b[39m     \u001b[38;5;66;03m# We try to see if we have a cached version (not up to date):\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m140\u001b[39m     resolved_file = \u001b[43mtry_to_load_from_cache\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath_or_repo_id\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfull_filename\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrevision\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrevision\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    141\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m resolved_file \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m resolved_file != _CACHED_NO_EXIST:\n\u001b[32m    142\u001b[39m         \u001b[38;5;28;01mreturn\u001b[39;00m resolved_file\n",
      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jeeva\\.conda\\envs\\textSummarizer\\Lib\\site-packages\\huggingface_hub\\utils\\_validators.py:106\u001b[39m, in \u001b[36mvalidate_hf_hub_args.<locals>._inner_fn\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m    101\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m arg_name, arg_value \u001b[38;5;129;01min\u001b[39;00m chain(\n\u001b[32m    102\u001b[39m     \u001b[38;5;28mzip\u001b[39m(signature.parameters, args),  \u001b[38;5;66;03m# Args values\u001b[39;00m\n\u001b[32m    103\u001b[39m     kwargs.items(),  \u001b[38;5;66;03m# Kwargs values\u001b[39;00m\n\u001b[32m    104\u001b[39m ):\n\u001b[32m    105\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m arg_name \u001b[38;5;129;01min\u001b[39;00m [\u001b[33m\"\u001b[39m\u001b[33mrepo_id\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mfrom_id\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mto_id\u001b[39m\u001b[33m\"\u001b[39m]:\n\u001b[32m--> \u001b[39m\u001b[32m106\u001b[39m         \u001b[43mvalidate_repo_id\u001b[49m\u001b[43m(\u001b[49m\u001b[43marg_value\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    108\u001b[39m     \u001b[38;5;28;01melif\u001b[39;00m arg_name == \u001b[33m\"\u001b[39m\u001b[33mtoken\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m arg_value \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m    109\u001b[39m         has_token = \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\jeeva\\.conda\\envs\\textSummarizer\\Lib\\site-packages\\huggingface_hub\\utils\\_validators.py:154\u001b[39m, in \u001b[36mvalidate_repo_id\u001b[39m\u001b[34m(repo_id)\u001b[39m\n\u001b[32m    151\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m HFValidationError(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mRepo id must be a string, not \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(repo_id)\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m: \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mrepo_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m.\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m    153\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m repo_id.count(\u001b[33m\"\u001b[39m\u001b[33m/\u001b[39m\u001b[33m\"\u001b[39m) > \u001b[32m1\u001b[39m:\n\u001b[32m--> \u001b[39m\u001b[32m154\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m HFValidationError(\n\u001b[32m    155\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mRepo id must be in the form \u001b[39m\u001b[33m'\u001b[39m\u001b[33mrepo_name\u001b[39m\u001b[33m'\u001b[39m\u001b[33m or \u001b[39m\u001b[33m'\u001b[39m\u001b[33mnamespace/repo_name\u001b[39m\u001b[33m'\u001b[39m\u001b[33m:\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    156\u001b[39m         \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mrepo_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m. Use `repo_type` argument if needed.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    157\u001b[39m     )\n\u001b[32m    159\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m REPO_ID_REGEX.match(repo_id):\n\u001b[32m    160\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m HFValidationError(\n\u001b[32m    161\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mRepo id must use alphanumeric chars or \u001b[39m\u001b[33m'\u001b[39m\u001b[33m-\u001b[39m\u001b[33m'\u001b[39m\u001b[33m, \u001b[39m\u001b[33m'\u001b[39m\u001b[33m_\u001b[39m\u001b[33m'\u001b[39m\u001b[33m, \u001b[39m\u001b[33m'\u001b[39m\u001b[33m.\u001b[39m\u001b[33m'\u001b[39m\u001b[33m, \u001b[39m\u001b[33m'\u001b[39m\u001b[33m--\u001b[39m\u001b[33m'\u001b[39m\u001b[33m and \u001b[39m\u001b[33m'\u001b[39m\u001b[33m..\u001b[39m\u001b[33m'\u001b[39m\u001b[33m are\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    162\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33m forbidden, \u001b[39m\u001b[33m'\u001b[39m\u001b[33m-\u001b[39m\u001b[33m'\u001b[39m\u001b[33m and \u001b[39m\u001b[33m'\u001b[39m\u001b[33m.\u001b[39m\u001b[33m'\u001b[39m\u001b[33m cannot start or end the name, max length is 96:\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    163\u001b[39m         \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mrepo_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    164\u001b[39m     )\n",
      "\u001b[31mHFValidationError\u001b[39m: Repo id must be in the form 'repo_name' or 'namespace/repo_name': 'artifacts/model_trainer/tokenizer_name'. Use `repo_type` argument if needed."
     ]
    }
   ],
   "source": [
    "try:\n",
    "    config = ConfigurationManager()\n",
    "    model_evaluation_config = config.get_model_evaluation_config()\n",
    "    model_evaluation_config = ModelEvaluation(config=model_evaluation_config)\n",
    "    model_evaluation_config.evaluate()\n",
    "except Exception as e:\n",
    "    raise e"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "textSummarizer",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}