Upload 8 files

NER_Customer_final.ipynb

@@ -0,0 +1,624 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Using TensorFlow backend.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import math\n",
+    "import numpy as np\n",
+    "from seqeval.metrics import f1_score\n",
+    "from seqeval.metrics import classification_report,accuracy_score,f1_score\n",
+    "import torch.nn.functional as F\n",
+    "import torch\n",
+    "import os\n",
+    "from tqdm import tqdm,trange\n",
+    "from torch.optim import Adam\n",
+    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
+    "from keras.preprocessing.sequence import pad_sequences\n",
+    "from pytorch_transformers import BertTokenizer, BertConfig\n",
+    "import torch\n",
+    "import torch.nn.functional as F\n",
+    "from pytorch_transformers import BertForTokenClassification, AdamW"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tag2idx = {'B-Address': 0,\n",
+    " 'B-Details': 1,\n",
+    " 'B-Auth_Status': 2,\n",
+    " 'B-Product_Name': 3,\n",
+    " 'B-Verification_Method': 4,\n",
+    " 'B-Category': 5,\n",
+    " 'B-Person': 6,\n",
+    " 'B-Complaint_Category': 7,\n",
+    " 'B-Field': 8,\n",
+    " 'B-Description': 9,\n",
+    " 'B-Divert': 10,\n",
+    " 'B-Auth_Method': 11,\n",
+    " 'B-Address_Status': 12,\n",
+    " 'B-Team': 13,\n",
+    " 'B-Service_Name': 14,\n",
+    " 'B-Status': 15,\n",
+    " 'I-Address': 16,\n",
+    " 'I-Details': 17,\n",
+    " 'I-Auth_Status': 18,\n",
+    " 'I-Product_Name': 19,\n",
+    " 'I-Verification_Method': 20,\n",
+    " 'I-Category': 21,\n",
+    " 'I-Person': 22,\n",
+    " 'I-Complaint_Category': 23,\n",
+    " 'I-Field': 24,\n",
+    " 'I-Description': 25,\n",
+    " 'I-Divert': 26,\n",
+    " 'I-Auth_Method': 27,\n",
+    " 'I-Address_Status': 28,\n",
+    " 'I-Team': 29,\n",
+    " 'I-Service_Name': 30,\n",
+    " 'I-Status': 31,\n",
+    " 'O': 32,\n",
+    " 'X': 33,\n",
+    " '[CLS]': 34,\n",
+    " '[SEP]': 35}\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tag2name={tag2idx[key] : key for key in tag2idx.keys()}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{0: 'B-Address',\n",
+       " 1: 'B-Details',\n",
+       " 2: 'B-Auth_Status',\n",
+       " 3: 'B-Product_Name',\n",
+       " 4: 'B-Verification_Method',\n",
+       " 5: 'B-Category',\n",
+       " 6: 'B-Person',\n",
+       " 7: 'B-Complaint_Category',\n",
+       " 8: 'B-Field',\n",
+       " 9: 'B-Description',\n",
+       " 10: 'B-Divert',\n",
+       " 11: 'B-Auth_Method',\n",
+       " 12: 'B-Address_Status',\n",
+       " 13: 'B-Team',\n",
+       " 14: 'B-Service_Name',\n",
+       " 15: 'B-Status',\n",
+       " 16: 'I-Address',\n",
+       " 17: 'I-Details',\n",
+       " 18: 'I-Auth_Status',\n",
+       " 19: 'I-Product_Name',\n",
+       " 20: 'I-Verification_Method',\n",
+       " 21: 'I-Category',\n",
+       " 22: 'I-Person',\n",
+       " 23: 'I-Complaint_Category',\n",
+       " 24: 'I-Field',\n",
+       " 25: 'I-Description',\n",
+       " 26: 'I-Divert',\n",
+       " 27: 'I-Auth_Method',\n",
+       " 28: 'I-Address_Status',\n",
+       " 29: 'I-Team',\n",
+       " 30: 'I-Service_Name',\n",
+       " 31: 'I-Status',\n",
+       " 32: 'O',\n",
+       " 33: 'X',\n",
+       " 34: '[CLS]',\n",
+       " 35: '[SEP]'}"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tag2name"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "BertForTokenClassification(\n",
+       "  (bert): BertModel(\n",
+       "    (embeddings): BertEmbeddings(\n",
+       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
+       "      (position_embeddings): Embedding(512, 768)\n",
+       "      (token_type_embeddings): Embedding(2, 768)\n",
+       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "      (dropout): Dropout(p=0.1, inplace=False)\n",
+       "    )\n",
+       "    (encoder): BertEncoder(\n",
+       "      (layer): ModuleList(\n",
+       "        (0): BertLayer(\n",
+       "          (attention): BertAttention(\n",
+       "            (self): BertSelfAttention(\n",
+       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "            (output): BertSelfOutput(\n",
+       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "          )\n",
+       "          (intermediate): BertIntermediate(\n",
+       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "          )\n",
+       "          (output): BertOutput(\n",
+       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "          )\n",
+       "        )\n",
+       "        (1): BertLayer(\n",
+       "          (attention): BertAttention(\n",
+       "            (self): BertSelfAttention(\n",
+       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "            (output): BertSelfOutput(\n",
+       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "          )\n",
+       "          (intermediate): BertIntermediate(\n",
+       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "          )\n",
+       "          (output): BertOutput(\n",
+       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "          )\n",
+       "        )\n",
+       "        (2): BertLayer(\n",
+       "          (attention): BertAttention(\n",
+       "            (self): BertSelfAttention(\n",
+       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "            (output): BertSelfOutput(\n",
+       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "          )\n",
+       "          (intermediate): BertIntermediate(\n",
+       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "          )\n",
+       "          (output): BertOutput(\n",
+       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "          )\n",
+       "        )\n",
+       "        (3): BertLayer(\n",
+       "          (attention): BertAttention(\n",
+       "            (self): BertSelfAttention(\n",
+       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "            (output): BertSelfOutput(\n",
+       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "          )\n",
+       "          (intermediate): BertIntermediate(\n",
+       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "          )\n",
+       "          (output): BertOutput(\n",
+       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "          )\n",
+       "        )\n",
+       "        (4): BertLayer(\n",
+       "          (attention): BertAttention(\n",
+       "            (self): BertSelfAttention(\n",
+       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "            (output): BertSelfOutput(\n",
+       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "          )\n",
+       "          (intermediate): BertIntermediate(\n",
+       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "          )\n",
+       "          (output): BertOutput(\n",
+       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "          )\n",
+       "        )\n",
+       "        (5): BertLayer(\n",
+       "          (attention): BertAttention(\n",
+       "            (self): BertSelfAttention(\n",
+       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "            (output): BertSelfOutput(\n",
+       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "          )\n",
+       "          (intermediate): BertIntermediate(\n",
+       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "          )\n",
+       "          (output): BertOutput(\n",
+       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "          )\n",
+       "        )\n",
+       "        (6): BertLayer(\n",
+       "          (attention): BertAttention(\n",
+       "            (self): BertSelfAttention(\n",
+       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "            (output): BertSelfOutput(\n",
+       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "          )\n",
+       "          (intermediate): BertIntermediate(\n",
+       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "          )\n",
+       "          (output): BertOutput(\n",
+       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "          )\n",
+       "        )\n",
+       "        (7): BertLayer(\n",
+       "          (attention): BertAttention(\n",
+       "            (self): BertSelfAttention(\n",
+       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "            (output): BertSelfOutput(\n",
+       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "          )\n",
+       "          (intermediate): BertIntermediate(\n",
+       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "          )\n",
+       "          (output): BertOutput(\n",
+       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "          )\n",
+       "        )\n",
+       "        (8): BertLayer(\n",
+       "          (attention): BertAttention(\n",
+       "            (self): BertSelfAttention(\n",
+       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "            (output): BertSelfOutput(\n",
+       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "          )\n",
+       "          (intermediate): BertIntermediate(\n",
+       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "          )\n",
+       "          (output): BertOutput(\n",
+       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "          )\n",
+       "        )\n",
+       "        (9): BertLayer(\n",
+       "          (attention): BertAttention(\n",
+       "            (self): BertSelfAttention(\n",
+       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "            (output): BertSelfOutput(\n",
+       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "          )\n",
+       "          (intermediate): BertIntermediate(\n",
+       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "          )\n",
+       "          (output): BertOutput(\n",
+       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "          )\n",
+       "        )\n",
+       "        (10): BertLayer(\n",
+       "          (attention): BertAttention(\n",
+       "            (self): BertSelfAttention(\n",
+       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "            (output): BertSelfOutput(\n",
+       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "          )\n",
+       "          (intermediate): BertIntermediate(\n",
+       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "          )\n",
+       "          (output): BertOutput(\n",
+       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "          )\n",
+       "        )\n",
+       "        (11): BertLayer(\n",
+       "          (attention): BertAttention(\n",
+       "            (self): BertSelfAttention(\n",
+       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "            (output): BertSelfOutput(\n",
+       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "              (dropout): Dropout(p=0.1, inplace=False)\n",
+       "            )\n",
+       "          )\n",
+       "          (intermediate): BertIntermediate(\n",
+       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
+       "          )\n",
+       "          (output): BertOutput(\n",
+       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
+       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
+       "            (dropout): Dropout(p=0.1, inplace=False)\n",
+       "          )\n",
+       "        )\n",
+       "      )\n",
+       "    )\n",
+       "    (pooler): BertPooler(\n",
+       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
+       "      (activation): Tanh()\n",
+       "    )\n",
+       "  )\n",
+       "  (dropout): Dropout(p=0.1, inplace=False)\n",
+       "  (classifier): Linear(in_features=768, out_features=36, bias=True)\n",
+       ")"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "config_path = \"Specify the path\"\n",
+    "model_weights_path = \"Specify the path\"\n",
+    "vocab_path = \"Specify the path\"\n",
+    "\n",
+    "# Load the configuration\n",
+    "config = BertConfig.from_json_file(config_path)\n",
+    "config.num_labels = len(tag2idx)  # Set the number of labels as per your tag2idx\n",
+    "\n",
+    "# Load the model with the config and weights\n",
+    "model = BertForTokenClassification(config)\n",
+    "model.load_state_dict(torch.load(model_weights_path, map_location=torch.device('cpu')))\n",
+    "\n",
+    "# Load the tokenizer (if needed)\n",
+    "tokenizer = BertTokenizer.from_pretrained(vocab_path)\n",
+    "\n",
+    "# Set the model to use CPU\n",
+    "device = torch.device('cpu')\n",
+    "model.to(device)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def predict_ner_labels(statement, model, tokenizer, tag2name, max_len=60, device='cpu'):\n",
+    "    # Tokenize and prepare the input\n",
+    "    tokenized_input = tokenizer.tokenize(statement)\n",
+    "    tokenized_input = ['[CLS]'] + tokenized_input + ['[SEP]']\n",
+    "    input_ids = tokenizer.convert_tokens_to_ids(tokenized_input)\n",
+    "    \n",
+    "    # Create attention mask\n",
+    "    attention_mask = [1] * len(input_ids)\n",
+    "    \n",
+    "    # Pad input IDs and attention mask\n",
+    "    padding_length = max_len - len(input_ids)\n",
+    "    input_ids += [0] * padding_length\n",
+    "    attention_mask += [0] * padding_length\n",
+    "    \n",
+    "    # Convert to tensors\n",
+    "    input_ids_tensor = torch.tensor([input_ids], dtype=torch.long).to(device)\n",
+    "    attention_mask_tensor = torch.tensor([attention_mask], dtype=torch.long).to(device)\n",
+    "    \n",
+    "    # Set model to evaluation mode and predict\n",
+    "    model.eval()\n",
+    "    with torch.no_grad():\n",
+    "        outputs = model(input_ids_tensor, attention_mask=attention_mask_tensor)\n",
+    "        logits = outputs[0]\n",
+    "        predicted_token_ids = torch.argmax(F.log_softmax(logits, dim=2), dim=2).cpu().numpy()\n",
+    "    \n",
+    "    # Convert predicted token IDs to labels\n",
+    "    predicted_labels = [tag2name[id_] for id_ in predicted_token_ids[0]]\n",
+    "    \n",
+    "    # Filter out special tokens ([CLS], [SEP], [PAD])\n",
+    "    filtered_tokens = []\n",
+    "    filtered_labels = []\n",
+    "    final_labels = []\n",
+    "    \n",
+    "    # For merging subwords\n",
+    "    current_word = \"\"\n",
+    "    current_label = \"\"\n",
+    "\n",
+    "    for token, label in zip(tokenized_input, predicted_labels):\n",
+    "        if token in [\"[CLS]\", \"[SEP]\", \"[PAD]\"]:\n",
+    "            continue\n",
+    "        \n",
+    "        # Check if the token is a subword (starts with '##')\n",
+    "        if token.startswith(\"##\"):\n",
+    "            current_word += token[2:]  # Append the subword without '##'\n",
+    "        else:\n",
+    "            if current_word:  # Append previous word with its label\n",
+    "                filtered_tokens.append(current_word)\n",
+    "                filtered_labels.append(current_label)\n",
+    "            current_word = token  # Start new word\n",
+    "            current_label = label\n",
+    "        \n",
+    "        # If it's a valid label (not 'X'), set the current label\n",
+    "        if label != 'X':\n",
+    "            current_label = label\n",
+    "    \n",
+    "    # Append the last word\n",
+    "    if current_word:\n",
+    "        filtered_tokens.append(current_word)\n",
+    "        filtered_labels.append(current_label)\n",
+    "\n",
+    "    # Return token-label pairs\n",
+    "    return list(zip(filtered_tokens, filtered_labels))\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "i: O\n",
+      "need: O\n",
+      "an: O\n",
+      "update: O\n",
+      "on: O\n",
+      "the: O\n",
+      "verification: O\n",
+      "status: O\n",
+      "for: O\n",
+      "my: O\n",
+      "new: O\n",
+      "address: O\n",
+      "in: O\n",
+      "jubilee: B-Address\n",
+      "hills: I-Address\n",
+      ",: I-Address\n",
+      "hyderabad: I-Address\n",
+      ",: O\n",
+      "as: O\n",
+      "per: O\n",
+      "instructions: O\n",
+      "from: O\n",
+      "the: O\n",
+      "finance: B-Team\n",
+      "team: I-Team\n",
+      "assigned: O\n",
+      "by: O\n",
+      "priya: B-Person\n",
+      ".: O\n"
+     ]
+    }
+   ],
+   "source": [
+    "statement = \"I need an update on the verification status for my new address in Jubilee Hills, Hyderabad, as per instructions from the Finance Team assigned by Priya.\"\n",
+    "predictions = predict_ner_labels(statement, model, tokenizer, tag2name)\n",
+    "for token, label in predictions:\n",
+    "    print(f\"{token}: {label}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

README.md

@@ -0,0 +1,97 @@
+# NER Customer Support Model
+
+This project builds and utilizes a Named Entity Recognition (NER) model tailored for customer support interactions. The model uses BERT and focuses on identifying customer-specific entities such as complaints, product names, and appointment information.
+
+## Introduction
+
+The goal of this NER model is to improve customer support interactions by recognizing specific entities from customer queries. This enables automated systems to efficiently interpret and route customer queries based on recognized entities.
+
+## Requirements
+
+This project requires Python 3.7.4 and specific libraries listed in the `requirements.txt` file. Notable dependencies include:
+- BERT (using the Transformers library)
+- PyTorch for model training and inference
+- seqeval for NER evaluation
+- pandas and numpy for data handling
+
+## Setup
+
+1. **Clone the repository and install dependencies**:
+   ```bash
+   pip install -r requirements.txt
+   ```
+
+2. **Download Pre-trained BERT model**:
+   Ensure you have a trained BERT model for token classification saved with configuration and weights. The model components should be available as separate files, such as `config.json`, `pytorch_model.bin`, and `vocab.txt`.
+
+## Usage
+
+1. **Load the Model and Tokenizer**:
+
+   Specify the paths to the configuration, model weights, and tokenizer files:
+
+   ```python
+   from transformers import BertForTokenClassification, BertConfig, BertTokenizer
+   import torch
+
+   # Specify paths
+   config_path = "path/to/config.json"
+   model_weights_path = "path/to/pytorch_model.bin"
+   vocab_path = "path/to/vocab.txt"
+
+   # Load config and model
+   config = BertConfig.from_json_file(config_path)
+   model = BertForTokenClassification(config)
+   model.load_state_dict(torch.load(model_weights_path, map_location=torch.device('cpu')))
+
+   # Load tokenizer
+   tokenizer = BertTokenizer.from_pretrained(vocab_path)
+
+   # Set device
+   device = torch.device('cpu')
+   model.to(device)
+   ```
+
+2. **Tag a Sentence**:
+
+   After loading the model, pass a sentence to be tagged for entities:
+
+   ```python
+   sentence = "Your sample customer query here."
+
+   # Tokenize and prepare inputs
+   inputs = tokenizer(sentence, return_tensors="pt", truncation=True, padding=True)
+   inputs = {key: val.to(device) for key, val in inputs.items()}
+
+   # Predict tags
+   with torch.no_grad():
+       outputs = model(**inputs)
+   logits = outputs.logits
+
+   # Convert predictions to tags
+   predictions = torch.argmax(logits, dim=-1).cpu().numpy()
+   tags = [config.id2label[label] for label in predictions[0]]
+
+   print("Tokens:", tokenizer.tokenize(sentence))
+   print("Tags:", tags)
+   ```
+
+## Results
+
+The model will output tokens and their corresponding tags for the provided sentence, allowing you to see which entities were recognized.
+
+## File Structure
+
+- `NER_Customer_final.ipynb`: The main notebook containing data preprocessing, model training, and evaluation.
+- `requirements.txt`: Lists required libraries.
+- `README.md`: This file.
+
+## Additional Notes
+
+Ensure that GPU support is enabled if available to speed up processing. The code is set to use CPU by default:
+
+```python
+device = torch.device('cpu')
+model.to(device)
+```
+

config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "finetuning_task": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "num_labels": 36,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 0,
+  "pruned_heads": {},
+  "torchscript": false,
+  "type_vocab_size": 2,
+  "vocab_size": 30522
+}

model_full_v3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e48996204b718b626a05012a4e6e9e92eeedee99d629dda11deea09ad5ac34d6
+size 438138609

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ffe5fb88ff543c79243578f5e27e9588a551aa629889beb59ea06e5da3fa646
+size 438118385

pytorch_model_v3.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff60de4460c85dd1120a7575b62a16deb1478c4f1749afdf4783a2c253e4f76d
+size 438118994

requirements.txt

@@ -0,0 +1,12 @@
+# Specify Python version
+python==3.7.4
+
+# Libraries used in the notebook
+pandas
+math
+numpy
+seqeval
+torch
+tqdm
+keras
+transformers