added the UI

app.py

@@ -0,0 +1,358 @@
+#Importing the necessary libraries
+import pandas as pd
+import torch
+from streamlit_option_menu import option_menu
+from plotting_helpers import (plot_top_5_products, plot_top_5_issues, plot_top_5_issues_in_product, plot_top_10_companies_complaints,
+                              plot_top_10_states_most_complaints, plot_top_10_states_least_complaints, complaints_by_year, 
+                              complaints_across_states)
+from transformers import pipeline
+import streamlit as st
+import pickle
+import warnings
+warnings.filterwarnings("ignore")
+
+# Setting page config
+st.set_page_config(page_title='CFPB Consumer Complaint Insights', page_icon='📋',
+                    layout="wide", initial_sidebar_state='expanded')
+
+@st.cache_data(show_spinner=False)
+def load_process_data():
+    df = pd.read_csv('complaints.csv')
+    df['Date received'] = pd.to_datetime(df['Date received'])
+
+    cols_to_consider = ['Product','Sub-product','Issue','Sub-issue','Consumer complaint narrative','Company public response','Company',
+                        'State', 'ZIP code', 'Date received']
+    df_new = df[cols_to_consider]
+
+    df_new = df_new.dropna()
+
+    product_map = {'Credit reporting or other personal consumer reports' : 'Credit Reporting',
+                'Credit reporting, credit repair services, or other personal consumer reports' : 'Credit Reporting',
+                'Payday loan, title loan, personal loan, or advance loan' : 'Loans / Mortgage',
+                'Payday loan, title loan, or personal loan' : 'Loans / Mortgage',
+                'Student loan' : 'Loans / Mortgage',
+                'Vehicle loan or lease' : 'Loans / Mortgage',
+                'Debt collection' : 'Debt collection',
+                'Credit card or prepaid card' : 'Credit/Prepaid Card',
+                'Credit card' : 'Credit/Prepaid Card',
+                'Prepaid card' : 'Credit/Prepaid Card',
+                'Mortgage' : 'Loans / Mortgage',
+                'Checking or savings account' : 'Checking or savings account'  
+                }
+
+    df_new.loc[:,'Product'] = df_new['Product'].map(product_map)
+
+
+    df_new['complaint length'] = df_new['Consumer complaint narrative'].apply(lambda x : len(x))
+    df_new = df_new[df_new['complaint length'] > 20]
+
+    complaints_to_exclude = ['See document attached', 'See the attached documents.', 'Incorrect information on my credit report', 'incorrect information on my credit report',
+    'please see attached file','Please see documents Attached','Incorrect information on my credit report.', 'Please see attached file', 'see attached',
+    'See attached', 'SEE ATTACHED DOCUMENTS', 'See Attached', 'SEE ATTACHMENT', 'SEE ATTACHMENTS', 
+    'XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX']
+
+    df_new = df_new[~df_new['Consumer complaint narrative'].isin(complaints_to_exclude)]
+
+    return df_new
+
+# Load the processed data
+df = load_process_data()
+
+# Loading the product classifier model
+device = "mps" if torch.backends.mps.is_available() else "cpu"
+# Initialize the pipeline for classifying product
+product_classifier = pipeline("text-classification", model="Mahesh9/distil-bert-fintuned-product-cfpb-complaints",
+                              max_length = 512, truncation = True, device = device)
+
+# Load sub-product classifier models
+with open('subproduct_prediction/models/Credit_Reporting_model.pkl', 'rb') as f:
+   trained_model_cr= pickle.load(f)
+with open('subproduct_prediction/models/Credit_Prepaid_Card_model.pkl', 'rb') as f:
+   trained_model_cp= pickle.load(f)
+with open('subproduct_prediction/models/Checking_saving_model.pkl', 'rb') as f:
+    trained_model_cs=pickle.load(f)
+with open('subproduct_prediction/models/loan_model.pkl', 'rb') as f:
+   trained_model_l= pickle.load(f)
+with open('subproduct_prediction/models/Debt_model.pkl', 'rb') as f:
+   trained_model_d= pickle.load(f)
+
+@st.cache_resource(show_spinner=False)
+# Define a function to select the appropriate subproduct prediction model based on the predicted product
+def select_subproduct_model(predicted_product):
+    if predicted_product == 'Credit Reporting' :
+        return trained_model_cr
+    elif predicted_product == 'Credit/Prepaid Card':
+        return trained_model_cp
+    elif predicted_product == 'Checking or savings account':
+        return trained_model_cs
+    elif predicted_product == 'Loans / Mortgage':
+        return trained_model_l
+    elif predicted_product == 'Debt collection':
+        return trained_model_d
+    else:
+        raise ValueError("Invalid predicted product category")
+
+# Loading the issue classifier model
+issue_classifier = pipeline("text-classification", model="Mahesh9/distil-bert-fintuned-issues-cfpb-complaints",
+                            max_length = 512, truncation = True, device = device)
+
+# Path to the models and their corresponding names
+issue_model_files = {
+    'trained_model_account_operations': 'subproduct_prediction/issue_models/account_operations_and_unauthorized_transaction_issues.pkl',
+    'trained_model_collect_debt': 'subproduct_prediction/issue_models/attempts_to_collect_debt_not_owed.pkl',
+    'trained_model_closing_account': 'subproduct_prediction/issue_models/closing_an_account.pkl',
+    'trained_model_closing_your_account': 'subproduct_prediction/issue_models/closing_your_account.pkl',
+    'trained_model_credit_report': 'subproduct_prediction/issue_models/credit_report_and_monitoring_issues.pkl',
+    'trained_model_lender': 'subproduct_prediction/issue_models/dealing_with_your_lender_or_servicer.pkl',
+    'trained_model_disputes': 'subproduct_prediction/issue_models/disputes_and_misrepresentations.pkl',
+    'trained_model_improper_use_report': 'subproduct_prediction/issue_models/improper_use_of_your_report.pkl',
+    'trained_model_incorrect_info': 'subproduct_prediction/issue_models/incorrect_information_on_your_report.pkl',
+    'trained_model_legal_and_threat': 'subproduct_prediction/issue_models/legal_and_threat_actions.pkl',
+    'trained_model_managing_account': 'subproduct_prediction/issue_models/managing_an_account.pkl',
+    'trained_model_payment_funds': 'subproduct_prediction/issue_models/payment_and_funds_management.pkl',
+    'trained_model_investigation_wrt_issue': 'subproduct_prediction/issue_models/problem_with_a_company\'s_investigation_into_an_existing_issue.pkl',
+    'trained_model_investigation_wrt_problem': 'subproduct_prediction/issue_models/problem_with_a_company\'s_investigation_into_an_existing_problem.pkl',
+    'trained_model_credit_investigation_wrt_problem': 'subproduct_prediction/issue_models/problem_with_a_credit_reporting_company\'s_investigation_into_an_existing_problem.pkl',
+    'trained_model_purchase_shown': 'subproduct_prediction/issue_models/problem_with_a_purchase_shown_on_your_statement.pkl',
+    'trained_model_notification_about_debt': 'subproduct_prediction/issue_models/written_notification_about_debt.pkl',
+}
+
+issue_models = {}
+
+for model_name, file_path in issue_model_files.items():
+    with open(file_path, 'rb') as f:
+        issue_models[model_name] = pickle.load(f)
+
+# Define a function to select the appropriate subissue prediction model based on the predicted issue
+def select_subissue_model(predicted_issue):
+    if predicted_issue == "Problem with a company's investigation into an existing problem":
+        return issue_models['trained_model_investigation_wrt_problem']
+        
+    elif predicted_issue == "Problem with a credit reporting company's investigation into an existing problem":
+        return issue_models['trained_model_credit_investigation_wrt_problem']
+
+    elif predicted_issue == "Problem with a company's investigation into an existing issue":
+        return issue_models['trained_model_investigation_wrt_issue']
+
+    elif predicted_issue == "Problem with a purchase shown on your statement":
+        return issue_models['trained_model_purchase_shown']
+
+    elif predicted_issue == "Incorrect information on your report":
+        return issue_models['trained_model_incorrect_info']
+        
+    elif predicted_issue == "Improper use of your report":
+        return issue_models['trained_model_improper_use_report']
+
+    elif predicted_issue == "Account Operations and Unauthorized Transaction Issues":
+        return issue_models['trained_model_account_operations']
+        
+    elif predicted_issue == "Payment and Funds Management":
+        return issue_models['trained_model_payment_funds']
+
+    elif predicted_issue == "Managing an account":
+        return issue_models['trained_model_managing_account']
+        
+    elif predicted_issue == "Attempts to collect debt not owed":
+        return issue_models['trained_model_collect_debt']
+
+    elif predicted_issue == "Written notification about debt":
+        return issue_models['trained_model_notification_about_debt']
+        
+    elif predicted_issue == "Dealing with your lender or servicer":
+        return issue_models['trained_model_lender']
+
+    elif predicted_issue == "Disputes and Misrepresentations":
+        return issue_models['trained_model_disputes']
+        
+    elif predicted_issue == "Closing your account":
+        return issue_models['trained_model_closing_your_account']
+
+    elif predicted_issue == "Closing an account":
+        return issue_models['trained_model_closing_account']
+        
+    elif predicted_issue == "Credit Report and Monitoring Issues":
+        return issue_models['trained_model_credit_report']
+
+    elif predicted_issue == "Legal and Threat Actions":
+        return issue_models['trained_model_legal_and_threat']
+        
+    else:
+        raise ValueError("Invalid predicted issue category")
+
+# Custom Headers for enhancing UI Text elements
+def custom_header(text, level=1):
+    if level == 1:
+        icon_url = "https://cfpb.github.io/design-system/images/uploads/logo_vertical_071720.png"
+        # Adjust the img style as needed (e.g., height, vertical alignment, margin)
+        st.markdown(f"""
+            <h1 style="text-align: center;">
+                <img src="{icon_url}" alt="Icon" style="vertical-align: middle; height: 112px; margin-right: -160px;">
+                <span style="color: #008000; font-family: 'Sans Serif';">{text}</span>
+            </h1>
+        """, unsafe_allow_html=True)
+        #st.markdown(f"<h1 style='text-align: center; color: #ef8236; font-family: sans serif;'>{text}</h1>", unsafe_allow_html=True)
+    elif level == 2:
+        st.markdown(f"<h2 style='text-align: center; color: #00749C; font-family: sans serif;'>{text}</h2>", unsafe_allow_html=True)
+    elif level == 3:
+        st.markdown(f"<h3 style='text-align: center; color: #00749C; font-family: sans serif;'>{text}</h3>", unsafe_allow_html=True)
+    elif level == 4:
+        st.markdown(f"<h5 style='text-align: center; color: #00749C; font-family: sans serif;'>{text}</h5>", unsafe_allow_html=True)
+    elif level == 5:
+        st.markdown(f"<h5 style='text-align: center; color: #f63366; font-family: sans serif;'>{text}</h5>", unsafe_allow_html=True)
+
+# Helper function for classifying the complaint
+def classify_complaint(narrative):
+    # Predict product category
+    predicted_product = product_classifier(narrative)[0]['label']
+    
+    # Load the appropriate subproduct prediction model
+    subproduct_model = select_subproduct_model(predicted_product)
+    # Predict subproduct category using the selected model
+    predicted_subproduct = subproduct_model.predict([narrative])[0]
+
+    # Predict the appropriate issue category using the narrative
+    predicted_issue = issue_classifier(narrative)[0]['label']
+    
+    # Load the appropriate subissue prediction model
+    subissue_model = select_subissue_model(predicted_issue)
+    # Predict subissue category using the selected model
+    predicted_subissue = subissue_model.predict([narrative])[0]
+    
+    return {
+        "Product" : predicted_product,
+        "Sub-product" : predicted_subproduct,
+        "Issue" : predicted_issue,
+        "Sub-issue" : predicted_subissue
+    }
+
+# Helper function to display key insights
+def plot_eda_charts(level):
+    if level == 1:
+        fig = complaints_by_year(df)
+        return fig 
+    
+    if level == 2:
+        fig = complaints_across_states(df)
+        return fig 
+    
+    if level == 3:
+        fig = plot_top_5_products(df)
+        return fig
+
+    if level == 4:
+        fig = plot_top_5_issues(df)
+        return fig
+    
+    if level == 5:
+        fig = plot_top_5_issues_in_product(df)
+        return fig
+    
+    if level == 6:
+        fig = plot_top_10_companies_complaints(df)
+        return fig
+    
+    if level == 7:
+        fig = plot_top_10_states_most_complaints(df)
+        return fig
+    
+    if level == 8:
+        fig = plot_top_10_states_least_complaints(df)
+        return fig
+    
+# Navigation setup
+with st.sidebar:
+    selected = option_menu(menu_title = "Navigate",
+                       options = ["Home", "Key Insights", "Complaint Classifier"]
+                       ,default_index = 0)
+    
+# Home Page
+if selected == "Home":
+    custom_header('CFPB Consumer Complaint Insights', level=1)
+    # Introduction
+    st.markdown("""
+    <div style='text-align: center; color: #333; font-size: 20px;'>
+        <p><strong>Uncover Consumer Trends and Automate Complaint Categorization with CFPB Insights</strong></p>
+    </div>
+    """, unsafe_allow_html=True)
+
+    st.write("\n")
+
+    # Project Motivation
+    st.markdown("""
+    ### :orange[Motivation]
+    Consumers can face challenges with financial products and services, leading to complaints that may not always be resolved directly with financial institutions. The **Consumer Financial Protection Bureau (CFPB)** acts as a mediator in these scenarios. However, consumers often struggle to categorize their complaints accurately, leading to inefficiencies in the resolution process. Our project aims to **facilitate faster resolution** by automatically categorizing complaints based on narrative descriptions, enhancing the efficiency of complaint management.
+    """, unsafe_allow_html=True)
+
+    # Impact
+    st.markdown("""
+    ### :green[Impact]
+    The implementation of our project has two primary impacts:
+    - **Ease for Consumers:** Automates the tagging of complaints into appropriate categories, reducing the need for consumers to understand complex financial product categories.
+    - **Industry Adoption:** Offers a streamlined approach to complaint handling that can be adopted by financial institutions beyond the CFPB, promoting consistency across the industry.
+    """, unsafe_allow_html=True)
+    # Complaint Classifier
+    st.markdown("""
+    #### :blue[Complaint Classifier]
+    Our dashboard features an innovative :rainbow[**Complaint Classifier**] that utilizes the narrative descriptions provided by consumers to categorize complaints into the correct product, issue, and sub-issue categories. This tool simplifies the submission process for consumers and enhances the efficiency of complaint resolution.
+    """, unsafe_allow_html=True)
+
+# Key Insights Page
+elif selected == "Key Insights":
+
+    headers = ["Evolution of complaints across years", "Complaints across US states",
+               "Top 5 Common Product Categories", "Top 5 Common Issue Categories",
+               "Top 5 Issues in Each Product Category", "Top 10 Companies with Most Complaints in 2023",
+               "Top 10 states with Most Complaints", "Top 10 states with Least Complaints"]
+
+    custom_header("Key Insights", level=1)
+    st.write("\n")
+    st.write("\n")
+    st.write("\n")
+
+    for i in range(0, len(headers), 2):
+        cols = st.columns(2)  # Create two columns
+        
+        with cols[0]:
+            custom_header(headers[i], level=4)
+            fig = plot_eda_charts(level=i+1)
+            st.plotly_chart(fig, use_container_width=True)
+        
+        if (i+1) < len(headers):
+            with cols[1]:
+                custom_header(headers[i+1], level=4)
+                fig = plot_eda_charts(level=i+2)
+                st.plotly_chart(fig, use_container_width=True)
+
+# Complaints Classifier Page
+elif selected == "Complaint Classifier":
+    custom_header("Complaint Classifier", level=2)
+    st.write("\n")
+    
+    # Using a key for the text_area widget to reference its current value
+    query = st.text_area("Enter your complaint:", placeholder="It is absurd that I have consistently made timely payments for this account and have never been overdue. I kindly request that you promptly update my account to reflect this accurately.", key="input_text")
+    if st.button("Classify Complaint"):
+        if query.strip():  # Check if the input is not empty
+            with st.spinner("Classifying Complaint..."):
+                result = classify_complaint(query)
+                if result:  # Check if the result is not empty
+                    st.success("Complaint Classification Results:")
+                    
+                    #Using HTML for better control over formatting
+                    st.markdown(f"""
+                    **Product:** :blue[{result.get("Product")}]<br>
+
+                    **Sub-product:** :green[{result.get("Sub-product")}]<br>
+
+                    **Issue:** :red[{result.get("Issue")}]<br>
+
+                    **Sub-issue:** :orange[{result.get("Sub-issue")}]<br>
+
+                    """, unsafe_allow_html=True)
+                    st.write("\n\n")
+                    st.header("", divider= 'rainbow')
+                else:
+                    st.error("Failed to classify the complaint. Please try again.")
+                #time.sleep(1)
+                st.balloons()  # Celebratory balloons on successful classification
+        else:
+            st.info("Please enter a complaint to classify.")

notebooks/.ipynb_checkpoints/Complaints preprocessing-Copy1-checkpoint.ipynb

@@ -0,0 +1,1061 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "cd6a338a-9a00-45f4-ac13-9ed131c9049e",
+   "metadata": {},
+   "source": [
+    "### Loading data (2023 year) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "2e8de3f1-6812-4c0d-bd56-32459911000e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "import plotly.express as px"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "ad45c437-7720-445e-8fa1-27d2b14b7bb5",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/tmp/ipykernel_42602/219708379.py:1: DtypeWarning: Columns (16) have mixed types. Specify dtype option on import or set low_memory=False.\n",
+      "  df = pd.read_csv('./complaints.csv')\n"
+     ]
+    }
+   ],
+   "source": [
+    "df = pd.read_csv('./complaints.csv')\n",
+    "df['Date received'] = pd.to_datetime(df['Date received'])\n",
+    "\n",
+    "cols_to_consider = ['Product','Sub-product','Issue','Sub-issue','Consumer complaint narrative','Company public response','Company',\n",
+    "                    'State', 'ZIP code', 'Date received']\n",
+    "df_new = df[cols_to_consider]\n",
+    "\n",
+    "df_new = df_new.dropna()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "id": "6df32835-7186-4c57-bffa-536f779636fe",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_2023 = df_new[df_new['Date received'].dt.year.isin([2023])].reset_index(drop=True)\n",
+    "\n",
+    "product_map = {'Credit reporting or other personal consumer reports' : 'Credit Reporting',\n",
+    "               'Credit reporting, credit repair services, or other personal consumer reports' : 'Credit Reporting',\n",
+    "               'Payday loan, title loan, personal loan, or advance loan' : 'Loans / Mortgage',\n",
+    "               'Payday loan, title loan, or personal loan' : 'Loans / Mortgage',\n",
+    "               'Student loan' : 'Loans / Mortgage',\n",
+    "               'Vehicle loan or lease' : 'Loans / Mortgage',\n",
+    "               'Debt collection' : 'Debt collection',\n",
+    "               'Credit card or prepaid card' : 'Credit/Prepaid Card',\n",
+    "               'Credit card' : 'Credit/Prepaid Card',\n",
+    "               'Prepaid card' : 'Credit/Prepaid Card',\n",
+    "               'Mortgage' : 'Loans / Mortgage',\n",
+    "               'Checking or savings account' : 'Checking or savings account'  \n",
+    "              }\n",
+    "\n",
+    "df_2023.loc[:,'Product'] = df_2023['Product'].map(product_map)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "id": "679ffbe3-a6ba-4f4d-bf65-0690794fb4e1",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Product</th>\n",
+       "      <th>Sub-product</th>\n",
+       "      <th>Issue</th>\n",
+       "      <th>Sub-issue</th>\n",
+       "      <th>Consumer complaint narrative</th>\n",
+       "      <th>Company public response</th>\n",
+       "      <th>Company</th>\n",
+       "      <th>State</th>\n",
+       "      <th>ZIP code</th>\n",
+       "      <th>Date received</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Checking or savings account</td>\n",
+       "      <td>Other banking product or service</td>\n",
+       "      <td>Opening an account</td>\n",
+       "      <td>Account opened without my consent or knowledge</td>\n",
+       "      <td>Date : XXXX XXXXo : XXXX XXXX XXXX / XXXX XXXX...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>WELLS FARGO &amp; COMPANY</td>\n",
+       "      <td>NC</td>\n",
+       "      <td>27513</td>\n",
+       "      <td>2023-12-29</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Problem with a company's investigation into an...</td>\n",
+       "      <td>Investigation took more than 30 days</td>\n",
+       "      <td>I have previously disputed this item with you ...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>Experian Information Solutions Inc.</td>\n",
+       "      <td>MN</td>\n",
+       "      <td>55124</td>\n",
+       "      <td>2023-12-29</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Debt collection</td>\n",
+       "      <td>Other debt</td>\n",
+       "      <td>Attempts to collect debt not owed</td>\n",
+       "      <td>Debt was result of identity theft</td>\n",
+       "      <td>I kindly request that you update my credit rep...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>Experian Information Solutions Inc.</td>\n",
+       "      <td>IL</td>\n",
+       "      <td>60621</td>\n",
+       "      <td>2023-12-28</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Debt collection</td>\n",
+       "      <td>Other debt</td>\n",
+       "      <td>Attempts to collect debt not owed</td>\n",
+       "      <td>Debt was result of identity theft</td>\n",
+       "      <td>I implore you to conduct a comprehensive inves...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>Experian Information Solutions Inc.</td>\n",
+       "      <td>NJ</td>\n",
+       "      <td>08723</td>\n",
+       "      <td>2023-12-28</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "      <td>In accordance with the Fair Credit Reporting A...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>TRANSUNION INTERMEDIATE HOLDINGS, INC.</td>\n",
+       "      <td>TX</td>\n",
+       "      <td>77377</td>\n",
+       "      <td>2023-11-27</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                       Product                       Sub-product  \\\n",
+       "0  Checking or savings account  Other banking product or service   \n",
+       "1             Credit Reporting                  Credit reporting   \n",
+       "2              Debt collection                        Other debt   \n",
+       "3              Debt collection                        Other debt   \n",
+       "4             Credit Reporting                  Credit reporting   \n",
+       "\n",
+       "                                               Issue  \\\n",
+       "0                                 Opening an account   \n",
+       "1  Problem with a company's investigation into an...   \n",
+       "2                  Attempts to collect debt not owed   \n",
+       "3                  Attempts to collect debt not owed   \n",
+       "4               Incorrect information on your report   \n",
+       "\n",
+       "                                        Sub-issue  \\\n",
+       "0  Account opened without my consent or knowledge   \n",
+       "1            Investigation took more than 30 days   \n",
+       "2               Debt was result of identity theft   \n",
+       "3               Debt was result of identity theft   \n",
+       "4             Information belongs to someone else   \n",
+       "\n",
+       "                        Consumer complaint narrative  \\\n",
+       "0  Date : XXXX XXXXo : XXXX XXXX XXXX / XXXX XXXX...   \n",
+       "1  I have previously disputed this item with you ...   \n",
+       "2  I kindly request that you update my credit rep...   \n",
+       "3  I implore you to conduct a comprehensive inves...   \n",
+       "4  In accordance with the Fair Credit Reporting A...   \n",
+       "\n",
+       "                             Company public response  \\\n",
+       "0  Company has responded to the consumer and the ...   \n",
+       "1  Company has responded to the consumer and the ...   \n",
+       "2  Company has responded to the consumer and the ...   \n",
+       "3  Company has responded to the consumer and the ...   \n",
+       "4  Company has responded to the consumer and the ...   \n",
+       "\n",
+       "                                  Company State ZIP code Date received  \n",
+       "0                   WELLS FARGO & COMPANY    NC    27513    2023-12-29  \n",
+       "1     Experian Information Solutions Inc.    MN    55124    2023-12-29  \n",
+       "2     Experian Information Solutions Inc.    IL    60621    2023-12-28  \n",
+       "3     Experian Information Solutions Inc.    NJ    08723    2023-12-28  \n",
+       "4  TRANSUNION INTERMEDIATE HOLDINGS, INC.    TX    77377    2023-11-27  "
+      ]
+     },
+     "execution_count": 30,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_2023.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "a85ec9b1-5de7-47f9-b204-4d42c8880bbb",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['Product', 'Sub-product', 'Issue', 'Sub-issue',\n",
+       "       'Consumer complaint narrative', 'Company public response', 'Company',\n",
+       "       'State', 'ZIP code', 'Date received'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 31,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_2023.columns"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0487636d-9663-4fdb-b219-f9e6be257b51",
+   "metadata": {},
+   "source": [
+    "### Complaint pre-processing"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "id": "e35208c6-020a-4fb9-8c9f-13fdeee44935",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_2023['complaint length'] = df_2023['Consumer complaint narrative'].apply(lambda x : len(x))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "id": "63deb9bb-d48a-460b-8edb-f66575ec1eaf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_2023 = df_2023[df_2023['complaint length'] > 20]\n",
+    "\n",
+    "complaints_to_exclude = ['See document attached', 'See the attached documents.', 'Incorrect information on my credit report', 'incorrect information on my credit report',\n",
+    "'please see attached file','Please see documents Attached','Incorrect information on my credit report.', 'Please see attached file', 'see attached',\n",
+    "'See attached', 'SEE ATTACHED DOCUMENTS', 'See Attached', 'SEE ATTACHMENT', 'SEE ATTACHMENTS', \n",
+    "'XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX']\n",
+    "\n",
+    "df_2023 = df_2023[~df_2023['Consumer complaint narrative'].isin(complaints_to_exclude)]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "492f8261-3e01-41d5-8f24-82bd289ee229",
+   "metadata": {},
+   "source": [
+    "### Categories consideration"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "id": "0be9e1f3-61aa-494a-bd0c-a6afeab5aacd",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(264968, 5)"
+      ]
+     },
+     "execution_count": 56,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_2023_subset = df_2023[['Consumer complaint narrative','Product','Sub-product','Issue','Sub-issue']]\n",
+    "df_2023_subset.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 57,
+   "id": "33e4e7e3-6661-48aa-aec2-b706fa64338d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Product\n",
+       "Credit Reporting               213403\n",
+       "Credit/Prepaid Card             16319\n",
+       "Checking or savings account     15143\n",
+       "Debt collection                 11767\n",
+       "Loans / Mortgage                 8336\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 57,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_2023_subset['Product'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 58,
+   "id": "dbc49ba8-f15a-4a4b-b018-d9d2273620ba",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sub_issues_to_consider = df_2023_subset['Sub-issue'].value_counts()[df_2023_subset['Sub-issue'].value_counts() > 500].index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 59,
+   "id": "746db565-e6ff-4ab2-bf92-d56088c0f2da",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "reduced_subissues = df_2023_subset[df_2023_subset['Sub-issue'].isin(sub_issues_to_consider)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 60,
+   "id": "0f786b1d-b139-40b5-ad53-639d8687d3b4",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(248065, 5)"
+      ]
+     },
+     "execution_count": 60,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "reduced_subissues.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 61,
+   "id": "f64515e8-ac65-4041-a201-a8576a86d7ad",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Sub-issue\n",
+       "Information belongs to someone else                                                      57877\n",
+       "Reporting company used your report improperly                                            48781\n",
+       "Their investigation did not fix an error on your report                                  45407\n",
+       "Credit inquiries on your report that you don't recognize                                 13150\n",
+       "Account status incorrect                                                                 10271\n",
+       "Account information incorrect                                                             9307\n",
+       "Was not notified of investigation status or results                                       9201\n",
+       "Investigation took more than 30 days                                                      8937\n",
+       "Personal information incorrect                                                            5900\n",
+       "Debt is not yours                                                                         2821\n",
+       "Deposits and withdrawals                                                                  2626\n",
+       "Credit card company isn't resolving a dispute about a purchase on your statement          2289\n",
+       "Didn't receive enough information to verify debt                                          1816\n",
+       "Debt was result of identity theft                                                         1761\n",
+       "Old information reappears or never goes away                                              1716\n",
+       "Difficulty submitting a dispute or getting information about a dispute over the phone     1709\n",
+       "Company closed your account                                                               1517\n",
+       "Problem using a debit or ATM card                                                         1503\n",
+       "Public record information inaccurate                                                      1389\n",
+       "Transaction was not authorized                                                            1378\n",
+       "Problem with personal statement of dispute                                                1361\n",
+       "Other problem getting your report or credit score                                         1112\n",
+       "Debt was paid                                                                              969\n",
+       "Card was charged for something you did not purchase with the card                          964\n",
+       "Banking errors                                                                             958\n",
+       "Funds not handled or disbursed as instructed                                               955\n",
+       "Overdrafts and overdraft fees                                                              951\n",
+       "Attempted to collect wrong amount                                                          885\n",
+       "Information is missing that should be on the report                                        881\n",
+       "Problem during payment process                                                             840\n",
+       "Fee problem                                                                                764\n",
+       "Problem with fees                                                                          749\n",
+       "Received bad information about your loan                                                   710\n",
+       "Other problem                                                                              701\n",
+       "Threatened or suggested your credit would be damaged                                       687\n",
+       "Funds not received from closed account                                                     673\n",
+       "Trouble with how payments are being handled                                                650\n",
+       "Didn't receive notice of right to dispute                                                  644\n",
+       "Can't close your account                                                                   598\n",
+       "Problem accessing account                                                                  561\n",
+       "Account opened as a result of fraud                                                        561\n",
+       "Problem canceling credit monitoring or identify theft protection service                   521\n",
+       "Card opened as result of identity theft or fraud                                           511\n",
+       "Billing problem                                                                            503\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 61,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "reduced_subissues['Sub-issue'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 62,
+   "id": "6204eb53-1a5b-457f-ab67-957d73f568af",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sub_products_to_consider = reduced_subissues['Sub-product'].value_counts()[reduced_subissues['Sub-product'].value_counts() > 100].index\n",
+    "final_df_2023 = reduced_subissues[reduced_subissues['Sub-product'].isin(sub_products_to_consider)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 63,
+   "id": "781850e8-cd50-4d08-87aa-8d86715cc2ef",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(247517, 5)"
+      ]
+     },
+     "execution_count": 63,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023.shape"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0ab4f91f-c938-4093-a299-b895ea13121a",
+   "metadata": {},
+   "source": [
+    "### Value counts"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 64,
+   "id": "17ddf55c-f824-4b2c-8059-07d02597a1cb",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Product\n",
+       "Credit Reporting               211695\n",
+       "Checking or savings account     12285\n",
+       "Credit/Prepaid Card             11975\n",
+       "Debt collection                  9380\n",
+       "Loans / Mortgage                 2182\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 64,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023['Product'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 65,
+   "id": "eae2d688-f706-4c31-9228-1ae7eadbf228",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Sub-product\n",
+       "Credit reporting                              210735\n",
+       "General-purpose credit card or charge card     10668\n",
+       "Checking account                               10409\n",
+       "Other debt                                      3041\n",
+       "I do not know                                   2316\n",
+       "Credit card debt                                1652\n",
+       "Federal student loan servicing                  1344\n",
+       "Store credit card                               1307\n",
+       "Medical debt                                    1053\n",
+       "Savings account                                  989\n",
+       "Other personal consumer report                   960\n",
+       "Loan                                             732\n",
+       "Other banking product or service                 725\n",
+       "Auto debt                                        581\n",
+       "Telecommunications debt                          419\n",
+       "Rental debt                                      179\n",
+       "CD (Certificate of Deposit)                      162\n",
+       "Mortgage debt                                    139\n",
+       "Conventional home mortgage                       106\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 65,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023['Sub-product'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 66,
+   "id": "61179ec3-f49f-4d2a-adde-738a0ff89371",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Issue\n",
+       "Incorrect information on your report                                                87200\n",
+       "Improper use of your report                                                         61868\n",
+       "Problem with a credit reporting company's investigation into an existing problem    45371\n",
+       "Problem with a company's investigation into an existing problem                     20985\n",
+       "Managing an account                                                                  7367\n",
+       "Attempts to collect debt not owed                                                    5453\n",
+       "Problem with a purchase shown on your statement                                      3253\n",
+       "Written notification about debt                                                      2404\n",
+       "Closing an account                                                                   1975\n",
+       "Problem with a lender or other company charging your account                         1378\n",
+       "Dealing with your lender or servicer                                                 1293\n",
+       "Unable to get your credit report or credit score                                     1109\n",
+       "Problem caused by your funds being low                                                951\n",
+       "False statements or representation                                                    861\n",
+       "Problem when making payments                                                          840\n",
+       "Closing your account                                                                  813\n",
+       "Fees or interest                                                                      749\n",
+       "Other features, terms, or problems                                                    701\n",
+       "Took or threatened to take negative or legal action                                   662\n",
+       "Opening an account                                                                    561\n",
+       "Getting a credit card                                                                 511\n",
+       "Credit monitoring or identity theft protection services                               495\n",
+       "Managing the loan or lease                                                            468\n",
+       "Problem with a company's investigation into an existing issue                         223\n",
+       "Identity theft protection or other monitoring services                                 26\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 66,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df['Issue'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 67,
+   "id": "928750bb-7324-480f-aaa1-a4438841399c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Sub-issue\n",
+       "Information belongs to someone else                                                      57850\n",
+       "Reporting company used your report improperly                                            48732\n",
+       "Their investigation did not fix an error on your report                                  45395\n",
+       "Credit inquiries on your report that you don't recognize                                 13136\n",
+       "Account status incorrect                                                                 10208\n",
+       "Account information incorrect                                                             9267\n",
+       "Was not notified of investigation status or results                                       9200\n",
+       "Investigation took more than 30 days                                                      8928\n",
+       "Personal information incorrect                                                            5900\n",
+       "Debt is not yours                                                                         2785\n",
+       "Deposits and withdrawals                                                                  2626\n",
+       "Credit card company isn't resolving a dispute about a purchase on your statement          2289\n",
+       "Didn't receive enough information to verify debt                                          1777\n",
+       "Debt was result of identity theft                                                         1727\n",
+       "Old information reappears or never goes away                                              1714\n",
+       "Difficulty submitting a dispute or getting information about a dispute over the phone     1704\n",
+       "Company closed your account                                                               1517\n",
+       "Problem using a debit or ATM card                                                         1503\n",
+       "Public record information inaccurate                                                      1384\n",
+       "Transaction was not authorized                                                            1378\n",
+       "Problem with personal statement of dispute                                                1352\n",
+       "Other problem getting your report or credit score                                         1109\n",
+       "Card was charged for something you did not purchase with the card                          964\n",
+       "Banking errors                                                                             958\n",
+       "Funds not handled or disbursed as instructed                                               955\n",
+       "Overdrafts and overdraft fees                                                              951\n",
+       "Debt was paid                                                                              941\n",
+       "Information is missing that should be on the report                                        877\n",
+       "Attempted to collect wrong amount                                                          861\n",
+       "Problem during payment process                                                             840\n",
+       "Fee problem                                                                                764\n",
+       "Problem with fees                                                                          749\n",
+       "Other problem                                                                              701\n",
+       "Received bad information about your loan                                                   677\n",
+       "Funds not received from closed account                                                     673\n",
+       "Threatened or suggested your credit would be damaged                                       662\n",
+       "Didn't receive notice of right to dispute                                                  627\n",
+       "Trouble with how payments are being handled                                                616\n",
+       "Can't close your account                                                                   598\n",
+       "Problem accessing account                                                                  561\n",
+       "Account opened as a result of fraud                                                        561\n",
+       "Problem canceling credit monitoring or identify theft protection service                   521\n",
+       "Card opened as result of identity theft or fraud                                           511\n",
+       "Billing problem                                                                            468\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 67,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023['Sub-issue'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fd91e57e-766c-4c4b-92c1-4b61469be9b4",
+   "metadata": {},
+   "source": [
+    "### Unique categories"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 68,
+   "id": "028803cd-86c0-4c8a-9fab-8f05ba6793a1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Unique Product offerings: 5\n",
+      "Unique Sub-product offerings: 19\n",
+      "Unique Issue offerings: 25\n",
+      "Unique Sub-issue offerings: 44\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(f\"Unique Product offerings: {final_df_2023['Product'].nunique()}\")\n",
+    "print(f\"Unique Sub-product offerings: {final_df_2023['Sub-product'].nunique()}\")\n",
+    "print(f\"Unique Issue offerings: {final_df_2023['Issue'].nunique()}\")\n",
+    "print(f\"Unique Sub-issue offerings: {final_df_2023['Sub-issue'].nunique()}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "06ea0454-ed84-450a-90f7-e7552ffc181f",
+   "metadata": {},
+   "source": [
+    "### Preparing the train and test splits"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 69,
+   "id": "267b771c-f944-443a-8048-c2f0097f4f29",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.model_selection import train_test_split"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 70,
+   "id": "eebed808-66b4-4fa8-a0ce-872b70d18106",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Consumer complaint narrative</th>\n",
+       "      <th>Product</th>\n",
+       "      <th>Sub-product</th>\n",
+       "      <th>Issue</th>\n",
+       "      <th>Sub-issue</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>I have previously disputed this item with you ...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Problem with a company's investigation into an...</td>\n",
+       "      <td>Investigation took more than 30 days</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>I kindly request that you update my credit rep...</td>\n",
+       "      <td>Debt collection</td>\n",
+       "      <td>Other debt</td>\n",
+       "      <td>Attempts to collect debt not owed</td>\n",
+       "      <td>Debt was result of identity theft</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>I implore you to conduct a comprehensive inves...</td>\n",
+       "      <td>Debt collection</td>\n",
+       "      <td>Other debt</td>\n",
+       "      <td>Attempts to collect debt not owed</td>\n",
+       "      <td>Debt was result of identity theft</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>In accordance with the Fair Credit Reporting A...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>In accordance with Fair c=Credit Reporting Act...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Improper use of your report</td>\n",
+       "      <td>Reporting company used your report improperly</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                        Consumer complaint narrative           Product  \\\n",
+       "1  I have previously disputed this item with you ...  Credit Reporting   \n",
+       "2  I kindly request that you update my credit rep...   Debt collection   \n",
+       "3  I implore you to conduct a comprehensive inves...   Debt collection   \n",
+       "4  In accordance with the Fair Credit Reporting A...  Credit Reporting   \n",
+       "5  In accordance with Fair c=Credit Reporting Act...  Credit Reporting   \n",
+       "\n",
+       "        Sub-product                                              Issue  \\\n",
+       "1  Credit reporting  Problem with a company's investigation into an...   \n",
+       "2        Other debt                  Attempts to collect debt not owed   \n",
+       "3        Other debt                  Attempts to collect debt not owed   \n",
+       "4  Credit reporting               Incorrect information on your report   \n",
+       "5  Credit reporting                        Improper use of your report   \n",
+       "\n",
+       "                                       Sub-issue  \n",
+       "1           Investigation took more than 30 days  \n",
+       "2              Debt was result of identity theft  \n",
+       "3              Debt was result of identity theft  \n",
+       "4            Information belongs to someone else  \n",
+       "5  Reporting company used your report improperly  "
+      ]
+     },
+     "execution_count": 70,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 86,
+   "id": "da025cda-f04e-4822-b100-855e981d632a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X = final_df_2023['Consumer complaint narrative']\n",
+    "y = final_df_2023[['Product','Sub-product','Issue','Sub-issue']]\n",
+    "\n",
+    "X_train, X_test, y_train, y_test = train_test_split(X,y,stratify=y['Product'],test_size=0.25,random_state=42)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 91,
+   "id": "d291102d-7136-4512-84c2-ba970b169cbf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_df = pd.concat([X_train,y_train],axis = 1).reset_index(drop = True)\n",
+    "test_df = pd.concat([X_test,y_test],axis = 1).reset_index(drop = True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 92,
+   "id": "0006636f-24cf-41dd-98cd-dc3a2b65432f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Consumer complaint narrative</th>\n",
+       "      <th>Product</th>\n",
+       "      <th>Sub-product</th>\n",
+       "      <th>Issue</th>\n",
+       "      <th>Sub-issue</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>The credit bureaus keep disrespecting the laws...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Problem with a company's investigation into an...</td>\n",
+       "      <td>Their investigation did not fix an error on yo...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>I sent in a complaint in XXXX of 2021 about so...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>I ordered a copy of my report and I found out ...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Problem with a credit reporting company's inve...</td>\n",
+       "      <td>Their investigation did not fix an error on yo...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>It appears that my credit file has been compro...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>I have never authorized, consented to nor bene...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                        Consumer complaint narrative           Product  \\\n",
+       "0  The credit bureaus keep disrespecting the laws...  Credit Reporting   \n",
+       "1  I sent in a complaint in XXXX of 2021 about so...  Credit Reporting   \n",
+       "2  I ordered a copy of my report and I found out ...  Credit Reporting   \n",
+       "3  It appears that my credit file has been compro...  Credit Reporting   \n",
+       "4  I have never authorized, consented to nor bene...  Credit Reporting   \n",
+       "\n",
+       "        Sub-product                                              Issue  \\\n",
+       "0  Credit reporting  Problem with a company's investigation into an...   \n",
+       "1  Credit reporting               Incorrect information on your report   \n",
+       "2  Credit reporting  Problem with a credit reporting company's inve...   \n",
+       "3  Credit reporting               Incorrect information on your report   \n",
+       "4  Credit reporting               Incorrect information on your report   \n",
+       "\n",
+       "                                           Sub-issue  \n",
+       "0  Their investigation did not fix an error on yo...  \n",
+       "1                Information belongs to someone else  \n",
+       "2  Their investigation did not fix an error on yo...  \n",
+       "3                Information belongs to someone else  \n",
+       "4                Information belongs to someone else  "
+      ]
+     },
+     "execution_count": 92,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 94,
+   "id": "724b3508-7e79-4526-a20f-3797250f9cf9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(185637, 5)"
+      ]
+     },
+     "execution_count": 94,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_df.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 95,
+   "id": "06972769-eddd-4ee7-9ebc-e6f587ad5366",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(61880, 5)"
+      ]
+     },
+     "execution_count": 95,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "test_df.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 99,
+   "id": "de358d80-fd59-4f9c-83ee-2264659f4b0f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "directory_to_save = './data_splits/'\n",
+    "\n",
+    "if not os.path.exists(directory_to_save):\n",
+    "    os.makedirs(directory_to_save)\n",
+    "\n",
+    "train_df.to_csv(directory_to_save + 'train-data-split.csv',index = False)\n",
+    "test_df.to_csv(directory_to_save + 'test-data-split.csv',index = False)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.19"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/.ipynb_checkpoints/Complaints preprocessing_new-checkpoint.ipynb

@@ -0,0 +1,1102 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "cd6a338a-9a00-45f4-ac13-9ed131c9049e",
+   "metadata": {},
+   "source": [
+    "### Loading data (2023 year) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "2e8de3f1-6812-4c0d-bd56-32459911000e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "import plotly.express as px"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "ad45c437-7720-445e-8fa1-27d2b14b7bb5",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/tmp/ipykernel_9929/219708379.py:1: DtypeWarning: Columns (16) have mixed types. Specify dtype option on import or set low_memory=False.\n",
+      "  df = pd.read_csv('./complaints.csv')\n"
+     ]
+    }
+   ],
+   "source": [
+    "df = pd.read_csv('./complaints.csv')\n",
+    "df['Date received'] = pd.to_datetime(df['Date received'])\n",
+    "\n",
+    "cols_to_consider = ['Product','Sub-product','Issue','Sub-issue','Consumer complaint narrative','Company public response','Company',\n",
+    "                    'State', 'ZIP code', 'Date received']\n",
+    "df_new = df[cols_to_consider]\n",
+    "\n",
+    "df_new = df_new.dropna()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "6df32835-7186-4c57-bffa-536f779636fe",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_2023 = df_new[df_new['Date received'].dt.year.isin([2023])].reset_index(drop=True)\n",
+    "\n",
+    "product_map = {'Credit reporting or other personal consumer reports' : 'Credit Reporting',\n",
+    "               'Credit reporting, credit repair services, or other personal consumer reports' : 'Credit Reporting',\n",
+    "               'Payday loan, title loan, personal loan, or advance loan' : 'Loans / Mortgage',\n",
+    "               'Payday loan, title loan, or personal loan' : 'Loans / Mortgage',\n",
+    "               'Student loan' : 'Loans / Mortgage',\n",
+    "               'Vehicle loan or lease' : 'Loans / Mortgage',\n",
+    "               'Debt collection' : 'Debt collection',\n",
+    "               'Credit card or prepaid card' : 'Credit/Prepaid Card',\n",
+    "               'Credit card' : 'Credit/Prepaid Card',\n",
+    "               'Prepaid card' : 'Credit/Prepaid Card',\n",
+    "               'Mortgage' : 'Loans / Mortgage',\n",
+    "               'Checking or savings account' : 'Checking or savings account'  \n",
+    "              }\n",
+    "\n",
+    "df_2023.loc[:,'Product'] = df_2023['Product'].map(product_map)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "679ffbe3-a6ba-4f4d-bf65-0690794fb4e1",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Product</th>\n",
+       "      <th>Sub-product</th>\n",
+       "      <th>Issue</th>\n",
+       "      <th>Sub-issue</th>\n",
+       "      <th>Consumer complaint narrative</th>\n",
+       "      <th>Company public response</th>\n",
+       "      <th>Company</th>\n",
+       "      <th>State</th>\n",
+       "      <th>ZIP code</th>\n",
+       "      <th>Date received</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Checking or savings account</td>\n",
+       "      <td>Other banking product or service</td>\n",
+       "      <td>Opening an account</td>\n",
+       "      <td>Account opened without my consent or knowledge</td>\n",
+       "      <td>Date : XXXX XXXXo : XXXX XXXX XXXX / XXXX XXXX...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>WELLS FARGO &amp; COMPANY</td>\n",
+       "      <td>NC</td>\n",
+       "      <td>27513</td>\n",
+       "      <td>2023-12-29</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Problem with a company's investigation into an...</td>\n",
+       "      <td>Investigation took more than 30 days</td>\n",
+       "      <td>I have previously disputed this item with you ...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>Experian Information Solutions Inc.</td>\n",
+       "      <td>MN</td>\n",
+       "      <td>55124</td>\n",
+       "      <td>2023-12-29</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Debt collection</td>\n",
+       "      <td>Other debt</td>\n",
+       "      <td>Attempts to collect debt not owed</td>\n",
+       "      <td>Debt was result of identity theft</td>\n",
+       "      <td>I kindly request that you update my credit rep...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>Experian Information Solutions Inc.</td>\n",
+       "      <td>IL</td>\n",
+       "      <td>60621</td>\n",
+       "      <td>2023-12-28</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Debt collection</td>\n",
+       "      <td>Other debt</td>\n",
+       "      <td>Attempts to collect debt not owed</td>\n",
+       "      <td>Debt was result of identity theft</td>\n",
+       "      <td>I implore you to conduct a comprehensive inves...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>Experian Information Solutions Inc.</td>\n",
+       "      <td>NJ</td>\n",
+       "      <td>08723</td>\n",
+       "      <td>2023-12-28</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "      <td>In accordance with the Fair Credit Reporting A...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>TRANSUNION INTERMEDIATE HOLDINGS, INC.</td>\n",
+       "      <td>TX</td>\n",
+       "      <td>77377</td>\n",
+       "      <td>2023-11-27</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                       Product                       Sub-product  \\\n",
+       "0  Checking or savings account  Other banking product or service   \n",
+       "1             Credit Reporting                  Credit reporting   \n",
+       "2              Debt collection                        Other debt   \n",
+       "3              Debt collection                        Other debt   \n",
+       "4             Credit Reporting                  Credit reporting   \n",
+       "\n",
+       "                                               Issue  \\\n",
+       "0                                 Opening an account   \n",
+       "1  Problem with a company's investigation into an...   \n",
+       "2                  Attempts to collect debt not owed   \n",
+       "3                  Attempts to collect debt not owed   \n",
+       "4               Incorrect information on your report   \n",
+       "\n",
+       "                                        Sub-issue  \\\n",
+       "0  Account opened without my consent or knowledge   \n",
+       "1            Investigation took more than 30 days   \n",
+       "2               Debt was result of identity theft   \n",
+       "3               Debt was result of identity theft   \n",
+       "4             Information belongs to someone else   \n",
+       "\n",
+       "                        Consumer complaint narrative  \\\n",
+       "0  Date : XXXX XXXXo : XXXX XXXX XXXX / XXXX XXXX...   \n",
+       "1  I have previously disputed this item with you ...   \n",
+       "2  I kindly request that you update my credit rep...   \n",
+       "3  I implore you to conduct a comprehensive inves...   \n",
+       "4  In accordance with the Fair Credit Reporting A...   \n",
+       "\n",
+       "                             Company public response  \\\n",
+       "0  Company has responded to the consumer and the ...   \n",
+       "1  Company has responded to the consumer and the ...   \n",
+       "2  Company has responded to the consumer and the ...   \n",
+       "3  Company has responded to the consumer and the ...   \n",
+       "4  Company has responded to the consumer and the ...   \n",
+       "\n",
+       "                                  Company State ZIP code Date received  \n",
+       "0                   WELLS FARGO & COMPANY    NC    27513    2023-12-29  \n",
+       "1     Experian Information Solutions Inc.    MN    55124    2023-12-29  \n",
+       "2     Experian Information Solutions Inc.    IL    60621    2023-12-28  \n",
+       "3     Experian Information Solutions Inc.    NJ    08723    2023-12-28  \n",
+       "4  TRANSUNION INTERMEDIATE HOLDINGS, INC.    TX    77377    2023-11-27  "
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_2023.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "a85ec9b1-5de7-47f9-b204-4d42c8880bbb",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['Product', 'Sub-product', 'Issue', 'Sub-issue',\n",
+       "       'Consumer complaint narrative', 'Company public response', 'Company',\n",
+       "       'State', 'ZIP code', 'Date received'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_2023.columns"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0487636d-9663-4fdb-b219-f9e6be257b51",
+   "metadata": {},
+   "source": [
+    "### Complaint pre-processing"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "e35208c6-020a-4fb9-8c9f-13fdeee44935",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_2023['complaint length'] = df_2023['Consumer complaint narrative'].apply(lambda x : len(x))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "63deb9bb-d48a-460b-8edb-f66575ec1eaf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_2023 = df_2023[df_2023['complaint length'] > 20]\n",
+    "\n",
+    "complaints_to_exclude = ['See document attached', 'See the attached documents.', 'Incorrect information on my credit report', 'incorrect information on my credit report',\n",
+    "'please see attached file','Please see documents Attached','Incorrect information on my credit report.', 'Please see attached file', 'see attached',\n",
+    "'See attached', 'SEE ATTACHED DOCUMENTS', 'See Attached', 'SEE ATTACHMENT', 'SEE ATTACHMENTS', \n",
+    "'XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX']\n",
+    "\n",
+    "df_2023 = df_2023[~df_2023['Consumer complaint narrative'].isin(complaints_to_exclude)]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "492f8261-3e01-41d5-8f24-82bd289ee229",
+   "metadata": {},
+   "source": [
+    "### Categories consideration"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "0be9e1f3-61aa-494a-bd0c-a6afeab5aacd",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(264968, 5)"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_2023_subset = df_2023[['Consumer complaint narrative','Product','Sub-product','Issue','Sub-issue']]\n",
+    "df_2023_subset.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "33e4e7e3-6661-48aa-aec2-b706fa64338d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Product\n",
+       "Credit Reporting               213403\n",
+       "Credit/Prepaid Card             16319\n",
+       "Checking or savings account     15143\n",
+       "Debt collection                 11767\n",
+       "Loans / Mortgage                 8336\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_2023_subset['Product'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "dbc49ba8-f15a-4a4b-b018-d9d2273620ba",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sub_issues_to_consider = df_2023_subset['Sub-issue'].value_counts()[df_2023_subset['Sub-issue'].value_counts() > 500].index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "746db565-e6ff-4ab2-bf92-d56088c0f2da",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "reduced_subissues = df_2023_subset[df_2023_subset['Sub-issue'].isin(sub_issues_to_consider)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "0f786b1d-b139-40b5-ad53-639d8687d3b4",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(248065, 5)"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "reduced_subissues.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "f64515e8-ac65-4041-a201-a8576a86d7ad",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Sub-issue\n",
+       "Information belongs to someone else                                                      57877\n",
+       "Reporting company used your report improperly                                            48781\n",
+       "Their investigation did not fix an error on your report                                  45407\n",
+       "Credit inquiries on your report that you don't recognize                                 13150\n",
+       "Account status incorrect                                                                 10271\n",
+       "Account information incorrect                                                             9307\n",
+       "Was not notified of investigation status or results                                       9201\n",
+       "Investigation took more than 30 days                                                      8937\n",
+       "Personal information incorrect                                                            5900\n",
+       "Debt is not yours                                                                         2821\n",
+       "Deposits and withdrawals                                                                  2626\n",
+       "Credit card company isn't resolving a dispute about a purchase on your statement          2289\n",
+       "Didn't receive enough information to verify debt                                          1816\n",
+       "Debt was result of identity theft                                                         1761\n",
+       "Old information reappears or never goes away                                              1716\n",
+       "Difficulty submitting a dispute or getting information about a dispute over the phone     1709\n",
+       "Company closed your account                                                               1517\n",
+       "Problem using a debit or ATM card                                                         1503\n",
+       "Public record information inaccurate                                                      1389\n",
+       "Transaction was not authorized                                                            1378\n",
+       "Problem with personal statement of dispute                                                1361\n",
+       "Other problem getting your report or credit score                                         1112\n",
+       "Debt was paid                                                                              969\n",
+       "Card was charged for something you did not purchase with the card                          964\n",
+       "Banking errors                                                                             958\n",
+       "Funds not handled or disbursed as instructed                                               955\n",
+       "Overdrafts and overdraft fees                                                              951\n",
+       "Attempted to collect wrong amount                                                          885\n",
+       "Information is missing that should be on the report                                        881\n",
+       "Problem during payment process                                                             840\n",
+       "Fee problem                                                                                764\n",
+       "Problem with fees                                                                          749\n",
+       "Received bad information about your loan                                                   710\n",
+       "Other problem                                                                              701\n",
+       "Threatened or suggested your credit would be damaged                                       687\n",
+       "Funds not received from closed account                                                     673\n",
+       "Trouble with how payments are being handled                                                650\n",
+       "Didn't receive notice of right to dispute                                                  644\n",
+       "Can't close your account                                                                   598\n",
+       "Problem accessing account                                                                  561\n",
+       "Account opened as a result of fraud                                                        561\n",
+       "Problem canceling credit monitoring or identify theft protection service                   521\n",
+       "Card opened as result of identity theft or fraud                                           511\n",
+       "Billing problem                                                                            503\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "reduced_subissues['Sub-issue'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "6204eb53-1a5b-457f-ab67-957d73f568af",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sub_products_to_consider = reduced_subissues['Sub-product'].value_counts()[reduced_subissues['Sub-product'].value_counts() > 100].index\n",
+    "final_df_2023 = reduced_subissues[reduced_subissues['Sub-product'].isin(sub_products_to_consider)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "781850e8-cd50-4d08-87aa-8d86715cc2ef",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(247517, 5)"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023.shape"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "563955e5-8b1b-4d67-a552-5d1b69ff8891",
+   "metadata": {},
+   "source": [
+    "### Issue categories grouping"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "8cb41375-d72e-4f90-bde1-6ff13af37082",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "issues_to_subissues = {}\n",
+    "for issue in final_df_2023['Issue'].value_counts().index:\n",
+    "    issues_to_subissues[issue] = list(final_df_2023[final_df_2023['Issue'] == issue]['Sub-issue'].value_counts().to_dict().keys())\n",
+    "\n",
+    "one_subissue = {key : value for key,value in issues_to_subissues.items() if len(issues_to_subissues[key]) == 1}\n",
+    "more_than_one_subissue = {key : value for key,value in issues_to_subissues.items() if len(issues_to_subissues[key]) > 1}\n",
+    "\n",
+    "existing_issue_mapping = {issue : issue for issue in more_than_one_subissue}\n",
+    "\n",
+    "issue_renaming = {\n",
+    "    'Problem with a lender or other company charging your account': 'Account Operations and Unauthorized Transaction Issues',\n",
+    "    'Opening an account': 'Account Operations and Unauthorized Transaction Issues',\n",
+    "    'Getting a credit card': 'Account Operations and Unauthorized Transaction Issues',\n",
+    "\n",
+    "    'Unable to get your credit report or credit score': 'Credit Report and Monitoring Issues',\n",
+    "    'Credit monitoring or identity theft protection services': 'Credit Report and Monitoring Issues',\n",
+    "    'Identity theft protection or other monitoring services': 'Credit Report and Monitoring Issues',\n",
+    "    \n",
+    "    'Problem caused by your funds being low': 'Payment and Funds Management',\n",
+    "    'Problem when making payments': 'Payment and Funds Management',\n",
+    "    'Managing the loan or lease': 'Payment and Funds Management',\n",
+    "\n",
+    "    'False statements or representation': 'Disputes and Misrepresentations',\n",
+    "    'Fees or interest': 'Disputes and Misrepresentations',\n",
+    "    'Other features, terms, or problems': 'Disputes and Misrepresentations',\n",
+    "\n",
+    "    'Took or threatened to take negative or legal action': 'Legal and Threat Actions'\n",
+    "}\n",
+    "\n",
+    "issues_mapping = {**issue_renaming, **existing_issue_mapping}\n",
+    "\n",
+    "final_df_2023.loc[:,'Issue'] = final_df_2023['Issue'].apply(lambda x : issues_mapping[x])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0ab4f91f-c938-4093-a299-b895ea13121a",
+   "metadata": {},
+   "source": [
+    "### Value counts"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "17ddf55c-f824-4b2c-8059-07d02597a1cb",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Product\n",
+       "Credit Reporting               211695\n",
+       "Checking or savings account     12285\n",
+       "Credit/Prepaid Card             11975\n",
+       "Debt collection                  9380\n",
+       "Loans / Mortgage                 2182\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023['Product'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "eae2d688-f706-4c31-9228-1ae7eadbf228",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Sub-product\n",
+       "Credit reporting                              210735\n",
+       "General-purpose credit card or charge card     10668\n",
+       "Checking account                               10409\n",
+       "Other debt                                      3041\n",
+       "I do not know                                   2316\n",
+       "Credit card debt                                1652\n",
+       "Federal student loan servicing                  1344\n",
+       "Store credit card                               1307\n",
+       "Medical debt                                    1053\n",
+       "Savings account                                  989\n",
+       "Other personal consumer report                   960\n",
+       "Loan                                             732\n",
+       "Other banking product or service                 725\n",
+       "Auto debt                                        581\n",
+       "Telecommunications debt                          419\n",
+       "Rental debt                                      179\n",
+       "CD (Certificate of Deposit)                      162\n",
+       "Mortgage debt                                    139\n",
+       "Conventional home mortgage                       106\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023['Sub-product'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "61179ec3-f49f-4d2a-adde-738a0ff89371",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Issue\n",
+       "Incorrect information on your report                                                87200\n",
+       "Improper use of your report                                                         61868\n",
+       "Problem with a credit reporting company's investigation into an existing problem    45371\n",
+       "Problem with a company's investigation into an existing problem                     20985\n",
+       "Managing an account                                                                  7367\n",
+       "Attempts to collect debt not owed                                                    5453\n",
+       "Problem with a purchase shown on your statement                                      3253\n",
+       "Account Operations and Unauthorized Transaction Issues                               2450\n",
+       "Written notification about debt                                                      2404\n",
+       "Disputes and Misrepresentations                                                      2311\n",
+       "Payment and Funds Management                                                         2259\n",
+       "Closing an account                                                                   1975\n",
+       "Credit Report and Monitoring Issues                                                  1630\n",
+       "Dealing with your lender or servicer                                                 1293\n",
+       "Closing your account                                                                  813\n",
+       "Legal and Threat Actions                                                              662\n",
+       "Problem with a company's investigation into an existing issue                         223\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023['Issue'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "928750bb-7324-480f-aaa1-a4438841399c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Sub-issue\n",
+       "Information belongs to someone else                                                      57850\n",
+       "Reporting company used your report improperly                                            48732\n",
+       "Their investigation did not fix an error on your report                                  45395\n",
+       "Credit inquiries on your report that you don't recognize                                 13136\n",
+       "Account status incorrect                                                                 10208\n",
+       "Account information incorrect                                                             9267\n",
+       "Was not notified of investigation status or results                                       9200\n",
+       "Investigation took more than 30 days                                                      8928\n",
+       "Personal information incorrect                                                            5900\n",
+       "Debt is not yours                                                                         2785\n",
+       "Deposits and withdrawals                                                                  2626\n",
+       "Credit card company isn't resolving a dispute about a purchase on your statement          2289\n",
+       "Didn't receive enough information to verify debt                                          1777\n",
+       "Debt was result of identity theft                                                         1727\n",
+       "Old information reappears or never goes away                                              1714\n",
+       "Difficulty submitting a dispute or getting information about a dispute over the phone     1704\n",
+       "Company closed your account                                                               1517\n",
+       "Problem using a debit or ATM card                                                         1503\n",
+       "Public record information inaccurate                                                      1384\n",
+       "Transaction was not authorized                                                            1378\n",
+       "Problem with personal statement of dispute                                                1352\n",
+       "Other problem getting your report or credit score                                         1109\n",
+       "Card was charged for something you did not purchase with the card                          964\n",
+       "Banking errors                                                                             958\n",
+       "Funds not handled or disbursed as instructed                                               955\n",
+       "Overdrafts and overdraft fees                                                              951\n",
+       "Debt was paid                                                                              941\n",
+       "Information is missing that should be on the report                                        877\n",
+       "Attempted to collect wrong amount                                                          861\n",
+       "Problem during payment process                                                             840\n",
+       "Fee problem                                                                                764\n",
+       "Problem with fees                                                                          749\n",
+       "Other problem                                                                              701\n",
+       "Received bad information about your loan                                                   677\n",
+       "Funds not received from closed account                                                     673\n",
+       "Threatened or suggested your credit would be damaged                                       662\n",
+       "Didn't receive notice of right to dispute                                                  627\n",
+       "Trouble with how payments are being handled                                                616\n",
+       "Can't close your account                                                                   598\n",
+       "Problem accessing account                                                                  561\n",
+       "Account opened as a result of fraud                                                        561\n",
+       "Problem canceling credit monitoring or identify theft protection service                   521\n",
+       "Card opened as result of identity theft or fraud                                           511\n",
+       "Billing problem                                                                            468\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023['Sub-issue'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fd91e57e-766c-4c4b-92c1-4b61469be9b4",
+   "metadata": {},
+   "source": [
+    "### Unique categories"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "028803cd-86c0-4c8a-9fab-8f05ba6793a1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Unique Product offerings: 5\n",
+      "Unique Sub-product offerings: 19\n",
+      "Unique Issue offerings: 17\n",
+      "Unique Sub-issue offerings: 44\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(f\"Unique Product offerings: {final_df_2023['Product'].nunique()}\")\n",
+    "print(f\"Unique Sub-product offerings: {final_df_2023['Sub-product'].nunique()}\")\n",
+    "print(f\"Unique Issue offerings: {final_df_2023['Issue'].nunique()}\")\n",
+    "print(f\"Unique Sub-issue offerings: {final_df_2023['Sub-issue'].nunique()}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "06ea0454-ed84-450a-90f7-e7552ffc181f",
+   "metadata": {},
+   "source": [
+    "### Preparing the train and test splits"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "267b771c-f944-443a-8048-c2f0097f4f29",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.model_selection import train_test_split"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "eebed808-66b4-4fa8-a0ce-872b70d18106",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Consumer complaint narrative</th>\n",
+       "      <th>Product</th>\n",
+       "      <th>Sub-product</th>\n",
+       "      <th>Issue</th>\n",
+       "      <th>Sub-issue</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>I have previously disputed this item with you ...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Problem with a company's investigation into an...</td>\n",
+       "      <td>Investigation took more than 30 days</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>I kindly request that you update my credit rep...</td>\n",
+       "      <td>Debt collection</td>\n",
+       "      <td>Other debt</td>\n",
+       "      <td>Attempts to collect debt not owed</td>\n",
+       "      <td>Debt was result of identity theft</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>I implore you to conduct a comprehensive inves...</td>\n",
+       "      <td>Debt collection</td>\n",
+       "      <td>Other debt</td>\n",
+       "      <td>Attempts to collect debt not owed</td>\n",
+       "      <td>Debt was result of identity theft</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>In accordance with the Fair Credit Reporting A...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>In accordance with Fair c=Credit Reporting Act...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Improper use of your report</td>\n",
+       "      <td>Reporting company used your report improperly</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                        Consumer complaint narrative           Product  \\\n",
+       "1  I have previously disputed this item with you ...  Credit Reporting   \n",
+       "2  I kindly request that you update my credit rep...   Debt collection   \n",
+       "3  I implore you to conduct a comprehensive inves...   Debt collection   \n",
+       "4  In accordance with the Fair Credit Reporting A...  Credit Reporting   \n",
+       "5  In accordance with Fair c=Credit Reporting Act...  Credit Reporting   \n",
+       "\n",
+       "        Sub-product                                              Issue  \\\n",
+       "1  Credit reporting  Problem with a company's investigation into an...   \n",
+       "2        Other debt                  Attempts to collect debt not owed   \n",
+       "3        Other debt                  Attempts to collect debt not owed   \n",
+       "4  Credit reporting               Incorrect information on your report   \n",
+       "5  Credit reporting                        Improper use of your report   \n",
+       "\n",
+       "                                       Sub-issue  \n",
+       "1           Investigation took more than 30 days  \n",
+       "2              Debt was result of identity theft  \n",
+       "3              Debt was result of identity theft  \n",
+       "4            Information belongs to someone else  \n",
+       "5  Reporting company used your report improperly  "
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "da025cda-f04e-4822-b100-855e981d632a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X = final_df_2023['Consumer complaint narrative']\n",
+    "y = final_df_2023[['Product','Sub-product','Issue','Sub-issue']]\n",
+    "\n",
+    "X_train, X_test, y_train, y_test = train_test_split(X,y,stratify=y['Product'],test_size=0.25,random_state=42)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "d291102d-7136-4512-84c2-ba970b169cbf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_df = pd.concat([X_train,y_train],axis = 1).reset_index(drop = True)\n",
+    "test_df = pd.concat([X_test,y_test],axis = 1).reset_index(drop = True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "0006636f-24cf-41dd-98cd-dc3a2b65432f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Consumer complaint narrative</th>\n",
+       "      <th>Product</th>\n",
+       "      <th>Sub-product</th>\n",
+       "      <th>Issue</th>\n",
+       "      <th>Sub-issue</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>The credit bureaus keep disrespecting the laws...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Problem with a company's investigation into an...</td>\n",
+       "      <td>Their investigation did not fix an error on yo...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>I sent in a complaint in XXXX of 2021 about so...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>I ordered a copy of my report and I found out ...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Problem with a credit reporting company's inve...</td>\n",
+       "      <td>Their investigation did not fix an error on yo...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>It appears that my credit file has been compro...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>I have never authorized, consented to nor bene...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                        Consumer complaint narrative           Product  \\\n",
+       "0  The credit bureaus keep disrespecting the laws...  Credit Reporting   \n",
+       "1  I sent in a complaint in XXXX of 2021 about so...  Credit Reporting   \n",
+       "2  I ordered a copy of my report and I found out ...  Credit Reporting   \n",
+       "3  It appears that my credit file has been compro...  Credit Reporting   \n",
+       "4  I have never authorized, consented to nor bene...  Credit Reporting   \n",
+       "\n",
+       "        Sub-product                                              Issue  \\\n",
+       "0  Credit reporting  Problem with a company's investigation into an...   \n",
+       "1  Credit reporting               Incorrect information on your report   \n",
+       "2  Credit reporting  Problem with a credit reporting company's inve...   \n",
+       "3  Credit reporting               Incorrect information on your report   \n",
+       "4  Credit reporting               Incorrect information on your report   \n",
+       "\n",
+       "                                           Sub-issue  \n",
+       "0  Their investigation did not fix an error on yo...  \n",
+       "1                Information belongs to someone else  \n",
+       "2  Their investigation did not fix an error on yo...  \n",
+       "3                Information belongs to someone else  \n",
+       "4                Information belongs to someone else  "
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "724b3508-7e79-4526-a20f-3797250f9cf9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(185637, 5)"
+      ]
+     },
+     "execution_count": 27,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_df.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "06972769-eddd-4ee7-9ebc-e6f587ad5366",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(61880, 5)"
+      ]
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "test_df.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "id": "de358d80-fd59-4f9c-83ee-2264659f4b0f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "directory_to_save = './data_splits/'\n",
+    "\n",
+    "if not os.path.exists(directory_to_save):\n",
+    "    os.makedirs(directory_to_save)\n",
+    "\n",
+    "train_df.to_csv(directory_to_save + 'train-data-split.csv',index = False)\n",
+    "test_df.to_csv(directory_to_save + 'test-data-split.csv',index = False)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/.ipynb_checkpoints/Data preprocessing-checkpoint.ipynb

@@ -0,0 +1,1069 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "cd6a338a-9a00-45f4-ac13-9ed131c9049e",
+   "metadata": {
+    "jp-MarkdownHeadingCollapsed": true
+   },
+   "source": [
+    "### Loading data (2023 year) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "2e8de3f1-6812-4c0d-bd56-32459911000e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "import plotly.express as px"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "ad45c437-7720-445e-8fa1-27d2b14b7bb5",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/tmp/ipykernel_42602/219708379.py:1: DtypeWarning: Columns (16) have mixed types. Specify dtype option on import or set low_memory=False.\n",
+      "  df = pd.read_csv('./complaints.csv')\n"
+     ]
+    }
+   ],
+   "source": [
+    "df = pd.read_csv('./complaints.csv')\n",
+    "df['Date received'] = pd.to_datetime(df['Date received'])\n",
+    "\n",
+    "cols_to_consider = ['Product','Sub-product','Issue','Sub-issue','Consumer complaint narrative','Company public response','Company',\n",
+    "                    'State', 'ZIP code', 'Date received']\n",
+    "df_new = df[cols_to_consider]\n",
+    "\n",
+    "df_new = df_new.dropna()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "id": "6df32835-7186-4c57-bffa-536f779636fe",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_2023 = df_new[df_new['Date received'].dt.year.isin([2023])].reset_index(drop=True)\n",
+    "\n",
+    "product_map = {'Credit reporting or other personal consumer reports' : 'Credit Reporting',\n",
+    "               'Credit reporting, credit repair services, or other personal consumer reports' : 'Credit Reporting',\n",
+    "               'Payday loan, title loan, personal loan, or advance loan' : 'Loans / Mortgage',\n",
+    "               'Payday loan, title loan, or personal loan' : 'Loans / Mortgage',\n",
+    "               'Student loan' : 'Loans / Mortgage',\n",
+    "               'Vehicle loan or lease' : 'Loans / Mortgage',\n",
+    "               'Debt collection' : 'Debt collection',\n",
+    "               'Credit card or prepaid card' : 'Credit/Prepaid Card',\n",
+    "               'Credit card' : 'Credit/Prepaid Card',\n",
+    "               'Prepaid card' : 'Credit/Prepaid Card',\n",
+    "               'Mortgage' : 'Loans / Mortgage',\n",
+    "               'Checking or savings account' : 'Checking or savings account'  \n",
+    "              }\n",
+    "\n",
+    "df_2023.loc[:,'Product'] = df_2023['Product'].map(product_map)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "id": "679ffbe3-a6ba-4f4d-bf65-0690794fb4e1",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Product</th>\n",
+       "      <th>Sub-product</th>\n",
+       "      <th>Issue</th>\n",
+       "      <th>Sub-issue</th>\n",
+       "      <th>Consumer complaint narrative</th>\n",
+       "      <th>Company public response</th>\n",
+       "      <th>Company</th>\n",
+       "      <th>State</th>\n",
+       "      <th>ZIP code</th>\n",
+       "      <th>Date received</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Checking or savings account</td>\n",
+       "      <td>Other banking product or service</td>\n",
+       "      <td>Opening an account</td>\n",
+       "      <td>Account opened without my consent or knowledge</td>\n",
+       "      <td>Date : XXXX XXXXo : XXXX XXXX XXXX / XXXX XXXX...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>WELLS FARGO &amp; COMPANY</td>\n",
+       "      <td>NC</td>\n",
+       "      <td>27513</td>\n",
+       "      <td>2023-12-29</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Problem with a company's investigation into an...</td>\n",
+       "      <td>Investigation took more than 30 days</td>\n",
+       "      <td>I have previously disputed this item with you ...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>Experian Information Solutions Inc.</td>\n",
+       "      <td>MN</td>\n",
+       "      <td>55124</td>\n",
+       "      <td>2023-12-29</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Debt collection</td>\n",
+       "      <td>Other debt</td>\n",
+       "      <td>Attempts to collect debt not owed</td>\n",
+       "      <td>Debt was result of identity theft</td>\n",
+       "      <td>I kindly request that you update my credit rep...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>Experian Information Solutions Inc.</td>\n",
+       "      <td>IL</td>\n",
+       "      <td>60621</td>\n",
+       "      <td>2023-12-28</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Debt collection</td>\n",
+       "      <td>Other debt</td>\n",
+       "      <td>Attempts to collect debt not owed</td>\n",
+       "      <td>Debt was result of identity theft</td>\n",
+       "      <td>I implore you to conduct a comprehensive inves...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>Experian Information Solutions Inc.</td>\n",
+       "      <td>NJ</td>\n",
+       "      <td>08723</td>\n",
+       "      <td>2023-12-28</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "      <td>In accordance with the Fair Credit Reporting A...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>TRANSUNION INTERMEDIATE HOLDINGS, INC.</td>\n",
+       "      <td>TX</td>\n",
+       "      <td>77377</td>\n",
+       "      <td>2023-11-27</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                       Product                       Sub-product  \\\n",
+       "0  Checking or savings account  Other banking product or service   \n",
+       "1             Credit Reporting                  Credit reporting   \n",
+       "2              Debt collection                        Other debt   \n",
+       "3              Debt collection                        Other debt   \n",
+       "4             Credit Reporting                  Credit reporting   \n",
+       "\n",
+       "                                               Issue  \\\n",
+       "0                                 Opening an account   \n",
+       "1  Problem with a company's investigation into an...   \n",
+       "2                  Attempts to collect debt not owed   \n",
+       "3                  Attempts to collect debt not owed   \n",
+       "4               Incorrect information on your report   \n",
+       "\n",
+       "                                        Sub-issue  \\\n",
+       "0  Account opened without my consent or knowledge   \n",
+       "1            Investigation took more than 30 days   \n",
+       "2               Debt was result of identity theft   \n",
+       "3               Debt was result of identity theft   \n",
+       "4             Information belongs to someone else   \n",
+       "\n",
+       "                        Consumer complaint narrative  \\\n",
+       "0  Date : XXXX XXXXo : XXXX XXXX XXXX / XXXX XXXX...   \n",
+       "1  I have previously disputed this item with you ...   \n",
+       "2  I kindly request that you update my credit rep...   \n",
+       "3  I implore you to conduct a comprehensive inves...   \n",
+       "4  In accordance with the Fair Credit Reporting A...   \n",
+       "\n",
+       "                             Company public response  \\\n",
+       "0  Company has responded to the consumer and the ...   \n",
+       "1  Company has responded to the consumer and the ...   \n",
+       "2  Company has responded to the consumer and the ...   \n",
+       "3  Company has responded to the consumer and the ...   \n",
+       "4  Company has responded to the consumer and the ...   \n",
+       "\n",
+       "                                  Company State ZIP code Date received  \n",
+       "0                   WELLS FARGO & COMPANY    NC    27513    2023-12-29  \n",
+       "1     Experian Information Solutions Inc.    MN    55124    2023-12-29  \n",
+       "2     Experian Information Solutions Inc.    IL    60621    2023-12-28  \n",
+       "3     Experian Information Solutions Inc.    NJ    08723    2023-12-28  \n",
+       "4  TRANSUNION INTERMEDIATE HOLDINGS, INC.    TX    77377    2023-11-27  "
+      ]
+     },
+     "execution_count": 30,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_2023.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "a85ec9b1-5de7-47f9-b204-4d42c8880bbb",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['Product', 'Sub-product', 'Issue', 'Sub-issue',\n",
+       "       'Consumer complaint narrative', 'Company public response', 'Company',\n",
+       "       'State', 'ZIP code', 'Date received'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 31,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_2023.columns"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0487636d-9663-4fdb-b219-f9e6be257b51",
+   "metadata": {
+    "jp-MarkdownHeadingCollapsed": true
+   },
+   "source": [
+    "### Complaint pre-processing"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "id": "e35208c6-020a-4fb9-8c9f-13fdeee44935",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_2023['complaint length'] = df_2023['Consumer complaint narrative'].apply(lambda x : len(x))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "id": "63deb9bb-d48a-460b-8edb-f66575ec1eaf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_2023 = df_2023[df_2023['complaint length'] > 20]\n",
+    "\n",
+    "complaints_to_exclude = ['See document attached', 'See the attached documents.', 'Incorrect information on my credit report', 'incorrect information on my credit report',\n",
+    "'please see attached file','Please see documents Attached','Incorrect information on my credit report.', 'Please see attached file', 'see attached',\n",
+    "'See attached', 'SEE ATTACHED DOCUMENTS', 'See Attached', 'SEE ATTACHMENT', 'SEE ATTACHMENTS', \n",
+    "'XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX']\n",
+    "\n",
+    "df_2023 = df_2023[~df_2023['Consumer complaint narrative'].isin(complaints_to_exclude)]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "492f8261-3e01-41d5-8f24-82bd289ee229",
+   "metadata": {
+    "jp-MarkdownHeadingCollapsed": true
+   },
+   "source": [
+    "### Categories consideration"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "id": "0be9e1f3-61aa-494a-bd0c-a6afeab5aacd",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(264968, 5)"
+      ]
+     },
+     "execution_count": 56,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_2023_subset = df_2023[['Consumer complaint narrative','Product','Sub-product','Issue','Sub-issue']]\n",
+    "df_2023_subset.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 57,
+   "id": "33e4e7e3-6661-48aa-aec2-b706fa64338d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Product\n",
+       "Credit Reporting               213403\n",
+       "Credit/Prepaid Card             16319\n",
+       "Checking or savings account     15143\n",
+       "Debt collection                 11767\n",
+       "Loans / Mortgage                 8336\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 57,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_2023_subset['Product'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 58,
+   "id": "dbc49ba8-f15a-4a4b-b018-d9d2273620ba",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sub_issues_to_consider = df_2023_subset['Sub-issue'].value_counts()[df_2023_subset['Sub-issue'].value_counts() > 500].index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 59,
+   "id": "746db565-e6ff-4ab2-bf92-d56088c0f2da",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "reduced_subissues = df_2023_subset[df_2023_subset['Sub-issue'].isin(sub_issues_to_consider)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 60,
+   "id": "0f786b1d-b139-40b5-ad53-639d8687d3b4",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(248065, 5)"
+      ]
+     },
+     "execution_count": 60,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "reduced_subissues.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 61,
+   "id": "f64515e8-ac65-4041-a201-a8576a86d7ad",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Sub-issue\n",
+       "Information belongs to someone else                                                      57877\n",
+       "Reporting company used your report improperly                                            48781\n",
+       "Their investigation did not fix an error on your report                                  45407\n",
+       "Credit inquiries on your report that you don't recognize                                 13150\n",
+       "Account status incorrect                                                                 10271\n",
+       "Account information incorrect                                                             9307\n",
+       "Was not notified of investigation status or results                                       9201\n",
+       "Investigation took more than 30 days                                                      8937\n",
+       "Personal information incorrect                                                            5900\n",
+       "Debt is not yours                                                                         2821\n",
+       "Deposits and withdrawals                                                                  2626\n",
+       "Credit card company isn't resolving a dispute about a purchase on your statement          2289\n",
+       "Didn't receive enough information to verify debt                                          1816\n",
+       "Debt was result of identity theft                                                         1761\n",
+       "Old information reappears or never goes away                                              1716\n",
+       "Difficulty submitting a dispute or getting information about a dispute over the phone     1709\n",
+       "Company closed your account                                                               1517\n",
+       "Problem using a debit or ATM card                                                         1503\n",
+       "Public record information inaccurate                                                      1389\n",
+       "Transaction was not authorized                                                            1378\n",
+       "Problem with personal statement of dispute                                                1361\n",
+       "Other problem getting your report or credit score                                         1112\n",
+       "Debt was paid                                                                              969\n",
+       "Card was charged for something you did not purchase with the card                          964\n",
+       "Banking errors                                                                             958\n",
+       "Funds not handled or disbursed as instructed                                               955\n",
+       "Overdrafts and overdraft fees                                                              951\n",
+       "Attempted to collect wrong amount                                                          885\n",
+       "Information is missing that should be on the report                                        881\n",
+       "Problem during payment process                                                             840\n",
+       "Fee problem                                                                                764\n",
+       "Problem with fees                                                                          749\n",
+       "Received bad information about your loan                                                   710\n",
+       "Other problem                                                                              701\n",
+       "Threatened or suggested your credit would be damaged                                       687\n",
+       "Funds not received from closed account                                                     673\n",
+       "Trouble with how payments are being handled                                                650\n",
+       "Didn't receive notice of right to dispute                                                  644\n",
+       "Can't close your account                                                                   598\n",
+       "Problem accessing account                                                                  561\n",
+       "Account opened as a result of fraud                                                        561\n",
+       "Problem canceling credit monitoring or identify theft protection service                   521\n",
+       "Card opened as result of identity theft or fraud                                           511\n",
+       "Billing problem                                                                            503\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 61,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "reduced_subissues['Sub-issue'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 62,
+   "id": "6204eb53-1a5b-457f-ab67-957d73f568af",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sub_products_to_consider = reduced_subissues['Sub-product'].value_counts()[reduced_subissues['Sub-product'].value_counts() > 100].index\n",
+    "final_df_2023 = reduced_subissues[reduced_subissues['Sub-product'].isin(sub_products_to_consider)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 63,
+   "id": "781850e8-cd50-4d08-87aa-8d86715cc2ef",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(247517, 5)"
+      ]
+     },
+     "execution_count": 63,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023.shape"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0ab4f91f-c938-4093-a299-b895ea13121a",
+   "metadata": {
+    "jp-MarkdownHeadingCollapsed": true
+   },
+   "source": [
+    "### Value counts"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 64,
+   "id": "17ddf55c-f824-4b2c-8059-07d02597a1cb",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Product\n",
+       "Credit Reporting               211695\n",
+       "Checking or savings account     12285\n",
+       "Credit/Prepaid Card             11975\n",
+       "Debt collection                  9380\n",
+       "Loans / Mortgage                 2182\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 64,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023['Product'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 65,
+   "id": "eae2d688-f706-4c31-9228-1ae7eadbf228",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Sub-product\n",
+       "Credit reporting                              210735\n",
+       "General-purpose credit card or charge card     10668\n",
+       "Checking account                               10409\n",
+       "Other debt                                      3041\n",
+       "I do not know                                   2316\n",
+       "Credit card debt                                1652\n",
+       "Federal student loan servicing                  1344\n",
+       "Store credit card                               1307\n",
+       "Medical debt                                    1053\n",
+       "Savings account                                  989\n",
+       "Other personal consumer report                   960\n",
+       "Loan                                             732\n",
+       "Other banking product or service                 725\n",
+       "Auto debt                                        581\n",
+       "Telecommunications debt                          419\n",
+       "Rental debt                                      179\n",
+       "CD (Certificate of Deposit)                      162\n",
+       "Mortgage debt                                    139\n",
+       "Conventional home mortgage                       106\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 65,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023['Sub-product'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 66,
+   "id": "61179ec3-f49f-4d2a-adde-738a0ff89371",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Issue\n",
+       "Incorrect information on your report                                                87200\n",
+       "Improper use of your report                                                         61868\n",
+       "Problem with a credit reporting company's investigation into an existing problem    45371\n",
+       "Problem with a company's investigation into an existing problem                     20985\n",
+       "Managing an account                                                                  7367\n",
+       "Attempts to collect debt not owed                                                    5453\n",
+       "Problem with a purchase shown on your statement                                      3253\n",
+       "Written notification about debt                                                      2404\n",
+       "Closing an account                                                                   1975\n",
+       "Problem with a lender or other company charging your account                         1378\n",
+       "Dealing with your lender or servicer                                                 1293\n",
+       "Unable to get your credit report or credit score                                     1109\n",
+       "Problem caused by your funds being low                                                951\n",
+       "False statements or representation                                                    861\n",
+       "Problem when making payments                                                          840\n",
+       "Closing your account                                                                  813\n",
+       "Fees or interest                                                                      749\n",
+       "Other features, terms, or problems                                                    701\n",
+       "Took or threatened to take negative or legal action                                   662\n",
+       "Opening an account                                                                    561\n",
+       "Getting a credit card                                                                 511\n",
+       "Credit monitoring or identity theft protection services                               495\n",
+       "Managing the loan or lease                                                            468\n",
+       "Problem with a company's investigation into an existing issue                         223\n",
+       "Identity theft protection or other monitoring services                                 26\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 66,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "demo['Issue'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 67,
+   "id": "928750bb-7324-480f-aaa1-a4438841399c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Sub-issue\n",
+       "Information belongs to someone else                                                      57850\n",
+       "Reporting company used your report improperly                                            48732\n",
+       "Their investigation did not fix an error on your report                                  45395\n",
+       "Credit inquiries on your report that you don't recognize                                 13136\n",
+       "Account status incorrect                                                                 10208\n",
+       "Account information incorrect                                                             9267\n",
+       "Was not notified of investigation status or results                                       9200\n",
+       "Investigation took more than 30 days                                                      8928\n",
+       "Personal information incorrect                                                            5900\n",
+       "Debt is not yours                                                                         2785\n",
+       "Deposits and withdrawals                                                                  2626\n",
+       "Credit card company isn't resolving a dispute about a purchase on your statement          2289\n",
+       "Didn't receive enough information to verify debt                                          1777\n",
+       "Debt was result of identity theft                                                         1727\n",
+       "Old information reappears or never goes away                                              1714\n",
+       "Difficulty submitting a dispute or getting information about a dispute over the phone     1704\n",
+       "Company closed your account                                                               1517\n",
+       "Problem using a debit or ATM card                                                         1503\n",
+       "Public record information inaccurate                                                      1384\n",
+       "Transaction was not authorized                                                            1378\n",
+       "Problem with personal statement of dispute                                                1352\n",
+       "Other problem getting your report or credit score                                         1109\n",
+       "Card was charged for something you did not purchase with the card                          964\n",
+       "Banking errors                                                                             958\n",
+       "Funds not handled or disbursed as instructed                                               955\n",
+       "Overdrafts and overdraft fees                                                              951\n",
+       "Debt was paid                                                                              941\n",
+       "Information is missing that should be on the report                                        877\n",
+       "Attempted to collect wrong amount                                                          861\n",
+       "Problem during payment process                                                             840\n",
+       "Fee problem                                                                                764\n",
+       "Problem with fees                                                                          749\n",
+       "Other problem                                                                              701\n",
+       "Received bad information about your loan                                                   677\n",
+       "Funds not received from closed account                                                     673\n",
+       "Threatened or suggested your credit would be damaged                                       662\n",
+       "Didn't receive notice of right to dispute                                                  627\n",
+       "Trouble with how payments are being handled                                                616\n",
+       "Can't close your account                                                                   598\n",
+       "Problem accessing account                                                                  561\n",
+       "Account opened as a result of fraud                                                        561\n",
+       "Problem canceling credit monitoring or identify theft protection service                   521\n",
+       "Card opened as result of identity theft or fraud                                           511\n",
+       "Billing problem                                                                            468\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 67,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023['Sub-issue'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fd91e57e-766c-4c4b-92c1-4b61469be9b4",
+   "metadata": {},
+   "source": [
+    "### Unique categories"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 68,
+   "id": "028803cd-86c0-4c8a-9fab-8f05ba6793a1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Unique Product offerings: 5\n",
+      "Unique Sub-product offerings: 19\n",
+      "Unique Issue offerings: 25\n",
+      "Unique Sub-issue offerings: 44\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(f\"Unique Product offerings: {final_df_2023['Product'].nunique()}\")\n",
+    "print(f\"Unique Sub-product offerings: {final_df_2023['Sub-product'].nunique()}\")\n",
+    "print(f\"Unique Issue offerings: {final_df_2023['Issue'].nunique()}\")\n",
+    "print(f\"Unique Sub-issue offerings: {final_df_2023['Sub-issue'].nunique()}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "06ea0454-ed84-450a-90f7-e7552ffc181f",
+   "metadata": {},
+   "source": [
+    "### Preparing the train and test splits"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 69,
+   "id": "267b771c-f944-443a-8048-c2f0097f4f29",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.model_selection import train_test_split"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 70,
+   "id": "eebed808-66b4-4fa8-a0ce-872b70d18106",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Consumer complaint narrative</th>\n",
+       "      <th>Product</th>\n",
+       "      <th>Sub-product</th>\n",
+       "      <th>Issue</th>\n",
+       "      <th>Sub-issue</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>I have previously disputed this item with you ...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Problem with a company's investigation into an...</td>\n",
+       "      <td>Investigation took more than 30 days</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>I kindly request that you update my credit rep...</td>\n",
+       "      <td>Debt collection</td>\n",
+       "      <td>Other debt</td>\n",
+       "      <td>Attempts to collect debt not owed</td>\n",
+       "      <td>Debt was result of identity theft</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>I implore you to conduct a comprehensive inves...</td>\n",
+       "      <td>Debt collection</td>\n",
+       "      <td>Other debt</td>\n",
+       "      <td>Attempts to collect debt not owed</td>\n",
+       "      <td>Debt was result of identity theft</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>In accordance with the Fair Credit Reporting A...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>In accordance with Fair c=Credit Reporting Act...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Improper use of your report</td>\n",
+       "      <td>Reporting company used your report improperly</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                        Consumer complaint narrative           Product  \\\n",
+       "1  I have previously disputed this item with you ...  Credit Reporting   \n",
+       "2  I kindly request that you update my credit rep...   Debt collection   \n",
+       "3  I implore you to conduct a comprehensive inves...   Debt collection   \n",
+       "4  In accordance with the Fair Credit Reporting A...  Credit Reporting   \n",
+       "5  In accordance with Fair c=Credit Reporting Act...  Credit Reporting   \n",
+       "\n",
+       "        Sub-product                                              Issue  \\\n",
+       "1  Credit reporting  Problem with a company's investigation into an...   \n",
+       "2        Other debt                  Attempts to collect debt not owed   \n",
+       "3        Other debt                  Attempts to collect debt not owed   \n",
+       "4  Credit reporting               Incorrect information on your report   \n",
+       "5  Credit reporting                        Improper use of your report   \n",
+       "\n",
+       "                                       Sub-issue  \n",
+       "1           Investigation took more than 30 days  \n",
+       "2              Debt was result of identity theft  \n",
+       "3              Debt was result of identity theft  \n",
+       "4            Information belongs to someone else  \n",
+       "5  Reporting company used your report improperly  "
+      ]
+     },
+     "execution_count": 70,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 86,
+   "id": "da025cda-f04e-4822-b100-855e981d632a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X = final_df_2023['Consumer complaint narrative']\n",
+    "y = final_df_2023[['Product','Sub-product','Issue','Sub-issue']]\n",
+    "\n",
+    "X_train, X_test, y_train, y_test = train_test_split(X,y,stratify=y['Product'],test_size=0.25,random_state=42)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 91,
+   "id": "d291102d-7136-4512-84c2-ba970b169cbf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_df = pd.concat([X_train,y_train],axis = 1).reset_index(drop = True)\n",
+    "test_df = pd.concat([X_test,y_test],axis = 1).reset_index(drop = True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 92,
+   "id": "0006636f-24cf-41dd-98cd-dc3a2b65432f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Consumer complaint narrative</th>\n",
+       "      <th>Product</th>\n",
+       "      <th>Sub-product</th>\n",
+       "      <th>Issue</th>\n",
+       "      <th>Sub-issue</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>The credit bureaus keep disrespecting the laws...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Problem with a company's investigation into an...</td>\n",
+       "      <td>Their investigation did not fix an error on yo...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>I sent in a complaint in XXXX of 2021 about so...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>I ordered a copy of my report and I found out ...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Problem with a credit reporting company's inve...</td>\n",
+       "      <td>Their investigation did not fix an error on yo...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>It appears that my credit file has been compro...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>I have never authorized, consented to nor bene...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                        Consumer complaint narrative           Product  \\\n",
+       "0  The credit bureaus keep disrespecting the laws...  Credit Reporting   \n",
+       "1  I sent in a complaint in XXXX of 2021 about so...  Credit Reporting   \n",
+       "2  I ordered a copy of my report and I found out ...  Credit Reporting   \n",
+       "3  It appears that my credit file has been compro...  Credit Reporting   \n",
+       "4  I have never authorized, consented to nor bene...  Credit Reporting   \n",
+       "\n",
+       "        Sub-product                                              Issue  \\\n",
+       "0  Credit reporting  Problem with a company's investigation into an...   \n",
+       "1  Credit reporting               Incorrect information on your report   \n",
+       "2  Credit reporting  Problem with a credit reporting company's inve...   \n",
+       "3  Credit reporting               Incorrect information on your report   \n",
+       "4  Credit reporting               Incorrect information on your report   \n",
+       "\n",
+       "                                           Sub-issue  \n",
+       "0  Their investigation did not fix an error on yo...  \n",
+       "1                Information belongs to someone else  \n",
+       "2  Their investigation did not fix an error on yo...  \n",
+       "3                Information belongs to someone else  \n",
+       "4                Information belongs to someone else  "
+      ]
+     },
+     "execution_count": 92,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 94,
+   "id": "724b3508-7e79-4526-a20f-3797250f9cf9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(185637, 5)"
+      ]
+     },
+     "execution_count": 94,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_df.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 95,
+   "id": "06972769-eddd-4ee7-9ebc-e6f587ad5366",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(61880, 5)"
+      ]
+     },
+     "execution_count": 95,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "test_df.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 99,
+   "id": "de358d80-fd59-4f9c-83ee-2264659f4b0f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "directory_to_save = './data_splits/'\n",
+    "\n",
+    "if not os.path.exists(directory_to_save):\n",
+    "    os.makedirs(directory_to_save)\n",
+    "\n",
+    "train_df.to_csv(directory_to_save + 'train-data-split.csv',index = False)\n",
+    "test_df.to_csv(directory_to_save + 'test-data-split.csv',index = False)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/.ipynb_checkpoints/Data split-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/.ipynb_checkpoints/Untitled-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/Data preprocessing.ipynb

@@ -0,0 +1,1102 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "cd6a338a-9a00-45f4-ac13-9ed131c9049e",
+   "metadata": {},
+   "source": [
+    "### Loading data (2023 year) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "2e8de3f1-6812-4c0d-bd56-32459911000e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "import plotly.express as px"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "ad45c437-7720-445e-8fa1-27d2b14b7bb5",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/tmp/ipykernel_9929/219708379.py:1: DtypeWarning: Columns (16) have mixed types. Specify dtype option on import or set low_memory=False.\n",
+      "  df = pd.read_csv('./complaints.csv')\n"
+     ]
+    }
+   ],
+   "source": [
+    "df = pd.read_csv('./complaints.csv')\n",
+    "df['Date received'] = pd.to_datetime(df['Date received'])\n",
+    "\n",
+    "cols_to_consider = ['Product','Sub-product','Issue','Sub-issue','Consumer complaint narrative','Company public response','Company',\n",
+    "                    'State', 'ZIP code', 'Date received']\n",
+    "df_new = df[cols_to_consider]\n",
+    "\n",
+    "df_new = df_new.dropna()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "6df32835-7186-4c57-bffa-536f779636fe",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_2023 = df_new[df_new['Date received'].dt.year.isin([2023])].reset_index(drop=True)\n",
+    "\n",
+    "product_map = {'Credit reporting or other personal consumer reports' : 'Credit Reporting',\n",
+    "               'Credit reporting, credit repair services, or other personal consumer reports' : 'Credit Reporting',\n",
+    "               'Payday loan, title loan, personal loan, or advance loan' : 'Loans / Mortgage',\n",
+    "               'Payday loan, title loan, or personal loan' : 'Loans / Mortgage',\n",
+    "               'Student loan' : 'Loans / Mortgage',\n",
+    "               'Vehicle loan or lease' : 'Loans / Mortgage',\n",
+    "               'Debt collection' : 'Debt collection',\n",
+    "               'Credit card or prepaid card' : 'Credit/Prepaid Card',\n",
+    "               'Credit card' : 'Credit/Prepaid Card',\n",
+    "               'Prepaid card' : 'Credit/Prepaid Card',\n",
+    "               'Mortgage' : 'Loans / Mortgage',\n",
+    "               'Checking or savings account' : 'Checking or savings account'  \n",
+    "              }\n",
+    "\n",
+    "df_2023.loc[:,'Product'] = df_2023['Product'].map(product_map)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "679ffbe3-a6ba-4f4d-bf65-0690794fb4e1",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Product</th>\n",
+       "      <th>Sub-product</th>\n",
+       "      <th>Issue</th>\n",
+       "      <th>Sub-issue</th>\n",
+       "      <th>Consumer complaint narrative</th>\n",
+       "      <th>Company public response</th>\n",
+       "      <th>Company</th>\n",
+       "      <th>State</th>\n",
+       "      <th>ZIP code</th>\n",
+       "      <th>Date received</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Checking or savings account</td>\n",
+       "      <td>Other banking product or service</td>\n",
+       "      <td>Opening an account</td>\n",
+       "      <td>Account opened without my consent or knowledge</td>\n",
+       "      <td>Date : XXXX XXXXo : XXXX XXXX XXXX / XXXX XXXX...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>WELLS FARGO &amp; COMPANY</td>\n",
+       "      <td>NC</td>\n",
+       "      <td>27513</td>\n",
+       "      <td>2023-12-29</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Problem with a company's investigation into an...</td>\n",
+       "      <td>Investigation took more than 30 days</td>\n",
+       "      <td>I have previously disputed this item with you ...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>Experian Information Solutions Inc.</td>\n",
+       "      <td>MN</td>\n",
+       "      <td>55124</td>\n",
+       "      <td>2023-12-29</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Debt collection</td>\n",
+       "      <td>Other debt</td>\n",
+       "      <td>Attempts to collect debt not owed</td>\n",
+       "      <td>Debt was result of identity theft</td>\n",
+       "      <td>I kindly request that you update my credit rep...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>Experian Information Solutions Inc.</td>\n",
+       "      <td>IL</td>\n",
+       "      <td>60621</td>\n",
+       "      <td>2023-12-28</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Debt collection</td>\n",
+       "      <td>Other debt</td>\n",
+       "      <td>Attempts to collect debt not owed</td>\n",
+       "      <td>Debt was result of identity theft</td>\n",
+       "      <td>I implore you to conduct a comprehensive inves...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>Experian Information Solutions Inc.</td>\n",
+       "      <td>NJ</td>\n",
+       "      <td>08723</td>\n",
+       "      <td>2023-12-28</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "      <td>In accordance with the Fair Credit Reporting A...</td>\n",
+       "      <td>Company has responded to the consumer and the ...</td>\n",
+       "      <td>TRANSUNION INTERMEDIATE HOLDINGS, INC.</td>\n",
+       "      <td>TX</td>\n",
+       "      <td>77377</td>\n",
+       "      <td>2023-11-27</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                       Product                       Sub-product  \\\n",
+       "0  Checking or savings account  Other banking product or service   \n",
+       "1             Credit Reporting                  Credit reporting   \n",
+       "2              Debt collection                        Other debt   \n",
+       "3              Debt collection                        Other debt   \n",
+       "4             Credit Reporting                  Credit reporting   \n",
+       "\n",
+       "                                               Issue  \\\n",
+       "0                                 Opening an account   \n",
+       "1  Problem with a company's investigation into an...   \n",
+       "2                  Attempts to collect debt not owed   \n",
+       "3                  Attempts to collect debt not owed   \n",
+       "4               Incorrect information on your report   \n",
+       "\n",
+       "                                        Sub-issue  \\\n",
+       "0  Account opened without my consent or knowledge   \n",
+       "1            Investigation took more than 30 days   \n",
+       "2               Debt was result of identity theft   \n",
+       "3               Debt was result of identity theft   \n",
+       "4             Information belongs to someone else   \n",
+       "\n",
+       "                        Consumer complaint narrative  \\\n",
+       "0  Date : XXXX XXXXo : XXXX XXXX XXXX / XXXX XXXX...   \n",
+       "1  I have previously disputed this item with you ...   \n",
+       "2  I kindly request that you update my credit rep...   \n",
+       "3  I implore you to conduct a comprehensive inves...   \n",
+       "4  In accordance with the Fair Credit Reporting A...   \n",
+       "\n",
+       "                             Company public response  \\\n",
+       "0  Company has responded to the consumer and the ...   \n",
+       "1  Company has responded to the consumer and the ...   \n",
+       "2  Company has responded to the consumer and the ...   \n",
+       "3  Company has responded to the consumer and the ...   \n",
+       "4  Company has responded to the consumer and the ...   \n",
+       "\n",
+       "                                  Company State ZIP code Date received  \n",
+       "0                   WELLS FARGO & COMPANY    NC    27513    2023-12-29  \n",
+       "1     Experian Information Solutions Inc.    MN    55124    2023-12-29  \n",
+       "2     Experian Information Solutions Inc.    IL    60621    2023-12-28  \n",
+       "3     Experian Information Solutions Inc.    NJ    08723    2023-12-28  \n",
+       "4  TRANSUNION INTERMEDIATE HOLDINGS, INC.    TX    77377    2023-11-27  "
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_2023.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "a85ec9b1-5de7-47f9-b204-4d42c8880bbb",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['Product', 'Sub-product', 'Issue', 'Sub-issue',\n",
+       "       'Consumer complaint narrative', 'Company public response', 'Company',\n",
+       "       'State', 'ZIP code', 'Date received'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_2023.columns"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0487636d-9663-4fdb-b219-f9e6be257b51",
+   "metadata": {},
+   "source": [
+    "### Complaint pre-processing"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "e35208c6-020a-4fb9-8c9f-13fdeee44935",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_2023['complaint length'] = df_2023['Consumer complaint narrative'].apply(lambda x : len(x))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "63deb9bb-d48a-460b-8edb-f66575ec1eaf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_2023 = df_2023[df_2023['complaint length'] > 20]\n",
+    "\n",
+    "complaints_to_exclude = ['See document attached', 'See the attached documents.', 'Incorrect information on my credit report', 'incorrect information on my credit report',\n",
+    "'please see attached file','Please see documents Attached','Incorrect information on my credit report.', 'Please see attached file', 'see attached',\n",
+    "'See attached', 'SEE ATTACHED DOCUMENTS', 'See Attached', 'SEE ATTACHMENT', 'SEE ATTACHMENTS', \n",
+    "'XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX XXXX']\n",
+    "\n",
+    "df_2023 = df_2023[~df_2023['Consumer complaint narrative'].isin(complaints_to_exclude)]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "492f8261-3e01-41d5-8f24-82bd289ee229",
+   "metadata": {},
+   "source": [
+    "### Categories consideration"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "0be9e1f3-61aa-494a-bd0c-a6afeab5aacd",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(264968, 5)"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_2023_subset = df_2023[['Consumer complaint narrative','Product','Sub-product','Issue','Sub-issue']]\n",
+    "df_2023_subset.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "33e4e7e3-6661-48aa-aec2-b706fa64338d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Product\n",
+       "Credit Reporting               213403\n",
+       "Credit/Prepaid Card             16319\n",
+       "Checking or savings account     15143\n",
+       "Debt collection                 11767\n",
+       "Loans / Mortgage                 8336\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_2023_subset['Product'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "dbc49ba8-f15a-4a4b-b018-d9d2273620ba",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sub_issues_to_consider = df_2023_subset['Sub-issue'].value_counts()[df_2023_subset['Sub-issue'].value_counts() > 500].index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "746db565-e6ff-4ab2-bf92-d56088c0f2da",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "reduced_subissues = df_2023_subset[df_2023_subset['Sub-issue'].isin(sub_issues_to_consider)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "0f786b1d-b139-40b5-ad53-639d8687d3b4",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(248065, 5)"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "reduced_subissues.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "f64515e8-ac65-4041-a201-a8576a86d7ad",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Sub-issue\n",
+       "Information belongs to someone else                                                      57877\n",
+       "Reporting company used your report improperly                                            48781\n",
+       "Their investigation did not fix an error on your report                                  45407\n",
+       "Credit inquiries on your report that you don't recognize                                 13150\n",
+       "Account status incorrect                                                                 10271\n",
+       "Account information incorrect                                                             9307\n",
+       "Was not notified of investigation status or results                                       9201\n",
+       "Investigation took more than 30 days                                                      8937\n",
+       "Personal information incorrect                                                            5900\n",
+       "Debt is not yours                                                                         2821\n",
+       "Deposits and withdrawals                                                                  2626\n",
+       "Credit card company isn't resolving a dispute about a purchase on your statement          2289\n",
+       "Didn't receive enough information to verify debt                                          1816\n",
+       "Debt was result of identity theft                                                         1761\n",
+       "Old information reappears or never goes away                                              1716\n",
+       "Difficulty submitting a dispute or getting information about a dispute over the phone     1709\n",
+       "Company closed your account                                                               1517\n",
+       "Problem using a debit or ATM card                                                         1503\n",
+       "Public record information inaccurate                                                      1389\n",
+       "Transaction was not authorized                                                            1378\n",
+       "Problem with personal statement of dispute                                                1361\n",
+       "Other problem getting your report or credit score                                         1112\n",
+       "Debt was paid                                                                              969\n",
+       "Card was charged for something you did not purchase with the card                          964\n",
+       "Banking errors                                                                             958\n",
+       "Funds not handled or disbursed as instructed                                               955\n",
+       "Overdrafts and overdraft fees                                                              951\n",
+       "Attempted to collect wrong amount                                                          885\n",
+       "Information is missing that should be on the report                                        881\n",
+       "Problem during payment process                                                             840\n",
+       "Fee problem                                                                                764\n",
+       "Problem with fees                                                                          749\n",
+       "Received bad information about your loan                                                   710\n",
+       "Other problem                                                                              701\n",
+       "Threatened or suggested your credit would be damaged                                       687\n",
+       "Funds not received from closed account                                                     673\n",
+       "Trouble with how payments are being handled                                                650\n",
+       "Didn't receive notice of right to dispute                                                  644\n",
+       "Can't close your account                                                                   598\n",
+       "Problem accessing account                                                                  561\n",
+       "Account opened as a result of fraud                                                        561\n",
+       "Problem canceling credit monitoring or identify theft protection service                   521\n",
+       "Card opened as result of identity theft or fraud                                           511\n",
+       "Billing problem                                                                            503\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "reduced_subissues['Sub-issue'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "6204eb53-1a5b-457f-ab67-957d73f568af",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sub_products_to_consider = reduced_subissues['Sub-product'].value_counts()[reduced_subissues['Sub-product'].value_counts() > 100].index\n",
+    "final_df_2023 = reduced_subissues[reduced_subissues['Sub-product'].isin(sub_products_to_consider)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "781850e8-cd50-4d08-87aa-8d86715cc2ef",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(247517, 5)"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023.shape"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "563955e5-8b1b-4d67-a552-5d1b69ff8891",
+   "metadata": {},
+   "source": [
+    "### Issue categories grouping"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "8cb41375-d72e-4f90-bde1-6ff13af37082",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "issues_to_subissues = {}\n",
+    "for issue in final_df_2023['Issue'].value_counts().index:\n",
+    "    issues_to_subissues[issue] = list(final_df_2023[final_df_2023['Issue'] == issue]['Sub-issue'].value_counts().to_dict().keys())\n",
+    "\n",
+    "one_subissue = {key : value for key,value in issues_to_subissues.items() if len(issues_to_subissues[key]) == 1}\n",
+    "more_than_one_subissue = {key : value for key,value in issues_to_subissues.items() if len(issues_to_subissues[key]) > 1}\n",
+    "\n",
+    "existing_issue_mapping = {issue : issue for issue in more_than_one_subissue}\n",
+    "\n",
+    "issue_renaming = {\n",
+    "    'Problem with a lender or other company charging your account': 'Account Operations and Unauthorized Transaction Issues',\n",
+    "    'Opening an account': 'Account Operations and Unauthorized Transaction Issues',\n",
+    "    'Getting a credit card': 'Account Operations and Unauthorized Transaction Issues',\n",
+    "\n",
+    "    'Unable to get your credit report or credit score': 'Credit Report and Monitoring Issues',\n",
+    "    'Credit monitoring or identity theft protection services': 'Credit Report and Monitoring Issues',\n",
+    "    'Identity theft protection or other monitoring services': 'Credit Report and Monitoring Issues',\n",
+    "    \n",
+    "    'Problem caused by your funds being low': 'Payment and Funds Management',\n",
+    "    'Problem when making payments': 'Payment and Funds Management',\n",
+    "    'Managing the loan or lease': 'Payment and Funds Management',\n",
+    "\n",
+    "    'False statements or representation': 'Disputes and Misrepresentations',\n",
+    "    'Fees or interest': 'Disputes and Misrepresentations',\n",
+    "    'Other features, terms, or problems': 'Disputes and Misrepresentations',\n",
+    "\n",
+    "    'Took or threatened to take negative or legal action': 'Legal and Threat Actions'\n",
+    "}\n",
+    "\n",
+    "issues_mapping = {**issue_renaming, **existing_issue_mapping}\n",
+    "\n",
+    "final_df_2023.loc[:,'Issue'] = final_df_2023['Issue'].apply(lambda x : issues_mapping[x])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0ab4f91f-c938-4093-a299-b895ea13121a",
+   "metadata": {},
+   "source": [
+    "### Value counts"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "17ddf55c-f824-4b2c-8059-07d02597a1cb",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Product\n",
+       "Credit Reporting               211695\n",
+       "Checking or savings account     12285\n",
+       "Credit/Prepaid Card             11975\n",
+       "Debt collection                  9380\n",
+       "Loans / Mortgage                 2182\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023['Product'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "eae2d688-f706-4c31-9228-1ae7eadbf228",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Sub-product\n",
+       "Credit reporting                              210735\n",
+       "General-purpose credit card or charge card     10668\n",
+       "Checking account                               10409\n",
+       "Other debt                                      3041\n",
+       "I do not know                                   2316\n",
+       "Credit card debt                                1652\n",
+       "Federal student loan servicing                  1344\n",
+       "Store credit card                               1307\n",
+       "Medical debt                                    1053\n",
+       "Savings account                                  989\n",
+       "Other personal consumer report                   960\n",
+       "Loan                                             732\n",
+       "Other banking product or service                 725\n",
+       "Auto debt                                        581\n",
+       "Telecommunications debt                          419\n",
+       "Rental debt                                      179\n",
+       "CD (Certificate of Deposit)                      162\n",
+       "Mortgage debt                                    139\n",
+       "Conventional home mortgage                       106\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023['Sub-product'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "61179ec3-f49f-4d2a-adde-738a0ff89371",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Issue\n",
+       "Incorrect information on your report                                                87200\n",
+       "Improper use of your report                                                         61868\n",
+       "Problem with a credit reporting company's investigation into an existing problem    45371\n",
+       "Problem with a company's investigation into an existing problem                     20985\n",
+       "Managing an account                                                                  7367\n",
+       "Attempts to collect debt not owed                                                    5453\n",
+       "Problem with a purchase shown on your statement                                      3253\n",
+       "Account Operations and Unauthorized Transaction Issues                               2450\n",
+       "Written notification about debt                                                      2404\n",
+       "Disputes and Misrepresentations                                                      2311\n",
+       "Payment and Funds Management                                                         2259\n",
+       "Closing an account                                                                   1975\n",
+       "Credit Report and Monitoring Issues                                                  1630\n",
+       "Dealing with your lender or servicer                                                 1293\n",
+       "Closing your account                                                                  813\n",
+       "Legal and Threat Actions                                                              662\n",
+       "Problem with a company's investigation into an existing issue                         223\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023['Issue'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "928750bb-7324-480f-aaa1-a4438841399c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Sub-issue\n",
+       "Information belongs to someone else                                                      57850\n",
+       "Reporting company used your report improperly                                            48732\n",
+       "Their investigation did not fix an error on your report                                  45395\n",
+       "Credit inquiries on your report that you don't recognize                                 13136\n",
+       "Account status incorrect                                                                 10208\n",
+       "Account information incorrect                                                             9267\n",
+       "Was not notified of investigation status or results                                       9200\n",
+       "Investigation took more than 30 days                                                      8928\n",
+       "Personal information incorrect                                                            5900\n",
+       "Debt is not yours                                                                         2785\n",
+       "Deposits and withdrawals                                                                  2626\n",
+       "Credit card company isn't resolving a dispute about a purchase on your statement          2289\n",
+       "Didn't receive enough information to verify debt                                          1777\n",
+       "Debt was result of identity theft                                                         1727\n",
+       "Old information reappears or never goes away                                              1714\n",
+       "Difficulty submitting a dispute or getting information about a dispute over the phone     1704\n",
+       "Company closed your account                                                               1517\n",
+       "Problem using a debit or ATM card                                                         1503\n",
+       "Public record information inaccurate                                                      1384\n",
+       "Transaction was not authorized                                                            1378\n",
+       "Problem with personal statement of dispute                                                1352\n",
+       "Other problem getting your report or credit score                                         1109\n",
+       "Card was charged for something you did not purchase with the card                          964\n",
+       "Banking errors                                                                             958\n",
+       "Funds not handled or disbursed as instructed                                               955\n",
+       "Overdrafts and overdraft fees                                                              951\n",
+       "Debt was paid                                                                              941\n",
+       "Information is missing that should be on the report                                        877\n",
+       "Attempted to collect wrong amount                                                          861\n",
+       "Problem during payment process                                                             840\n",
+       "Fee problem                                                                                764\n",
+       "Problem with fees                                                                          749\n",
+       "Other problem                                                                              701\n",
+       "Received bad information about your loan                                                   677\n",
+       "Funds not received from closed account                                                     673\n",
+       "Threatened or suggested your credit would be damaged                                       662\n",
+       "Didn't receive notice of right to dispute                                                  627\n",
+       "Trouble with how payments are being handled                                                616\n",
+       "Can't close your account                                                                   598\n",
+       "Problem accessing account                                                                  561\n",
+       "Account opened as a result of fraud                                                        561\n",
+       "Problem canceling credit monitoring or identify theft protection service                   521\n",
+       "Card opened as result of identity theft or fraud                                           511\n",
+       "Billing problem                                                                            468\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023['Sub-issue'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fd91e57e-766c-4c4b-92c1-4b61469be9b4",
+   "metadata": {},
+   "source": [
+    "### Unique categories"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "028803cd-86c0-4c8a-9fab-8f05ba6793a1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Unique Product offerings: 5\n",
+      "Unique Sub-product offerings: 19\n",
+      "Unique Issue offerings: 17\n",
+      "Unique Sub-issue offerings: 44\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(f\"Unique Product offerings: {final_df_2023['Product'].nunique()}\")\n",
+    "print(f\"Unique Sub-product offerings: {final_df_2023['Sub-product'].nunique()}\")\n",
+    "print(f\"Unique Issue offerings: {final_df_2023['Issue'].nunique()}\")\n",
+    "print(f\"Unique Sub-issue offerings: {final_df_2023['Sub-issue'].nunique()}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "06ea0454-ed84-450a-90f7-e7552ffc181f",
+   "metadata": {},
+   "source": [
+    "### Preparing the train and test splits"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "267b771c-f944-443a-8048-c2f0097f4f29",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.model_selection import train_test_split"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "eebed808-66b4-4fa8-a0ce-872b70d18106",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Consumer complaint narrative</th>\n",
+       "      <th>Product</th>\n",
+       "      <th>Sub-product</th>\n",
+       "      <th>Issue</th>\n",
+       "      <th>Sub-issue</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>I have previously disputed this item with you ...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Problem with a company's investigation into an...</td>\n",
+       "      <td>Investigation took more than 30 days</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>I kindly request that you update my credit rep...</td>\n",
+       "      <td>Debt collection</td>\n",
+       "      <td>Other debt</td>\n",
+       "      <td>Attempts to collect debt not owed</td>\n",
+       "      <td>Debt was result of identity theft</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>I implore you to conduct a comprehensive inves...</td>\n",
+       "      <td>Debt collection</td>\n",
+       "      <td>Other debt</td>\n",
+       "      <td>Attempts to collect debt not owed</td>\n",
+       "      <td>Debt was result of identity theft</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>In accordance with the Fair Credit Reporting A...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>In accordance with Fair c=Credit Reporting Act...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Improper use of your report</td>\n",
+       "      <td>Reporting company used your report improperly</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                        Consumer complaint narrative           Product  \\\n",
+       "1  I have previously disputed this item with you ...  Credit Reporting   \n",
+       "2  I kindly request that you update my credit rep...   Debt collection   \n",
+       "3  I implore you to conduct a comprehensive inves...   Debt collection   \n",
+       "4  In accordance with the Fair Credit Reporting A...  Credit Reporting   \n",
+       "5  In accordance with Fair c=Credit Reporting Act...  Credit Reporting   \n",
+       "\n",
+       "        Sub-product                                              Issue  \\\n",
+       "1  Credit reporting  Problem with a company's investigation into an...   \n",
+       "2        Other debt                  Attempts to collect debt not owed   \n",
+       "3        Other debt                  Attempts to collect debt not owed   \n",
+       "4  Credit reporting               Incorrect information on your report   \n",
+       "5  Credit reporting                        Improper use of your report   \n",
+       "\n",
+       "                                       Sub-issue  \n",
+       "1           Investigation took more than 30 days  \n",
+       "2              Debt was result of identity theft  \n",
+       "3              Debt was result of identity theft  \n",
+       "4            Information belongs to someone else  \n",
+       "5  Reporting company used your report improperly  "
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "final_df_2023.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "da025cda-f04e-4822-b100-855e981d632a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X = final_df_2023['Consumer complaint narrative']\n",
+    "y = final_df_2023[['Product','Sub-product','Issue','Sub-issue']]\n",
+    "\n",
+    "X_train, X_test, y_train, y_test = train_test_split(X,y,stratify=y['Product'],test_size=0.25,random_state=42)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "d291102d-7136-4512-84c2-ba970b169cbf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_df = pd.concat([X_train,y_train],axis = 1).reset_index(drop = True)\n",
+    "test_df = pd.concat([X_test,y_test],axis = 1).reset_index(drop = True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "0006636f-24cf-41dd-98cd-dc3a2b65432f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Consumer complaint narrative</th>\n",
+       "      <th>Product</th>\n",
+       "      <th>Sub-product</th>\n",
+       "      <th>Issue</th>\n",
+       "      <th>Sub-issue</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>The credit bureaus keep disrespecting the laws...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Problem with a company's investigation into an...</td>\n",
+       "      <td>Their investigation did not fix an error on yo...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>I sent in a complaint in XXXX of 2021 about so...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>I ordered a copy of my report and I found out ...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Problem with a credit reporting company's inve...</td>\n",
+       "      <td>Their investigation did not fix an error on yo...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>It appears that my credit file has been compro...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>I have never authorized, consented to nor bene...</td>\n",
+       "      <td>Credit Reporting</td>\n",
+       "      <td>Credit reporting</td>\n",
+       "      <td>Incorrect information on your report</td>\n",
+       "      <td>Information belongs to someone else</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                        Consumer complaint narrative           Product  \\\n",
+       "0  The credit bureaus keep disrespecting the laws...  Credit Reporting   \n",
+       "1  I sent in a complaint in XXXX of 2021 about so...  Credit Reporting   \n",
+       "2  I ordered a copy of my report and I found out ...  Credit Reporting   \n",
+       "3  It appears that my credit file has been compro...  Credit Reporting   \n",
+       "4  I have never authorized, consented to nor bene...  Credit Reporting   \n",
+       "\n",
+       "        Sub-product                                              Issue  \\\n",
+       "0  Credit reporting  Problem with a company's investigation into an...   \n",
+       "1  Credit reporting               Incorrect information on your report   \n",
+       "2  Credit reporting  Problem with a credit reporting company's inve...   \n",
+       "3  Credit reporting               Incorrect information on your report   \n",
+       "4  Credit reporting               Incorrect information on your report   \n",
+       "\n",
+       "                                           Sub-issue  \n",
+       "0  Their investigation did not fix an error on yo...  \n",
+       "1                Information belongs to someone else  \n",
+       "2  Their investigation did not fix an error on yo...  \n",
+       "3                Information belongs to someone else  \n",
+       "4                Information belongs to someone else  "
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "724b3508-7e79-4526-a20f-3797250f9cf9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(185637, 5)"
+      ]
+     },
+     "execution_count": 27,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_df.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "06972769-eddd-4ee7-9ebc-e6f587ad5366",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(61880, 5)"
+      ]
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "test_df.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "id": "de358d80-fd59-4f9c-83ee-2264659f4b0f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "directory_to_save = './data_splits/'\n",
+    "\n",
+    "if not os.path.exists(directory_to_save):\n",
+    "    os.makedirs(directory_to_save)\n",
+    "\n",
+    "train_df.to_csv(directory_to_save + 'train-data-split.csv',index = False)\n",
+    "test_df.to_csv(directory_to_save + 'test-data-split.csv',index = False)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.19"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

plotting_helpers.py

@@ -0,0 +1,254 @@
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+import seaborn as sns
+import warnings
+warnings.filterwarnings('ignore')
+
+# State abbreviation to full name mapping
+state_mapping = {
+    'FL': 'Florida', 'CA': 'California', 'TX': 'Texas', 'GA': 'Georgia',
+    'NY': 'New York', 'IL': 'Illinois', 'PA': 'Pennsylvania', 'NC': 'North Carolina',
+    'NJ': 'New Jersey', 'MD': 'Maryland', 'VA': 'Virginia', 'OH': 'Ohio',
+    'MI': 'Michigan', 'SC': 'South Carolina', 'AZ': 'Arizona', 'TN': 'Tennessee',
+    'NV': 'Nevada', 'LA': 'Louisiana', 'AL': 'Alabama', 'MO': 'Missouri',
+    'MA': 'Massachusetts', 'IN': 'Indiana', 'AR': 'Arkansas', 'WA': 'Washington',
+    'CO': 'Colorado', 'MS': 'Mississippi', 'CT': 'Connecticut', 'MN': 'Minnesota',
+    'WI': 'Wisconsin', 'KY': 'Kentucky', 'UT': 'Utah', 'DE': 'Delaware',
+    'OR': 'Oregon', 'OK': 'Oklahoma', 'DC': 'District of Columbia', 'KS': 'Kansas',
+    'IA': 'Iowa', 'NM': 'New Mexico', 'NE': 'Nebraska', 'HI': 'Hawaii',
+    'RI': 'Rhode Island', 'ID': 'Idaho', 'WV': 'West Virginia', 'NH': 'New Hampshire',
+    'ME': 'Maine', 'MT': 'Montana', 'ND': 'North Dakota', 'AK': 'Alaska',
+    'SD': 'South Dakota', 'WY': 'Wyoming', 'VT': 'Vermont'
+    # Removed territories and minor outlying islands not listed as states
+}
+
+# Function to plot top n most common categories
+def plot_top_n(df, column, title, n=5, palette_name=None):
+    # Generate a color sequence from the seaborn palette
+    color_sequence = sns.color_palette(palette_name, n_colors=n).as_hex() if palette_name else None
+    
+    # Get top n most common values in the specified column
+    counts = df[column].value_counts().reset_index()
+    counts.columns = [column, 'Count']
+    top_n = counts.head(n)
+    
+    # Create a horizontal bar plot with the seaborn color sequence and remove the legend
+    fig = px.bar(top_n, y=column, x='Count', orientation='h', 
+                 color=column, color_discrete_sequence=color_sequence)
+    fig.update_layout(showlegend=False)
+    return fig
+
+# 1. Plotting top 5 most common products
+def plot_top_5_products(df_new):
+    # df_new = load_process_data(df)
+    fig = plot_top_n(df_new, 'Product', 'Top 5 Most Common Products')
+    return fig
+
+# 2. Plotting Top 5 common issues
+def plot_top_5_issues(df_new):
+    # df_new = load_process_data(df)
+    fig = plot_top_n(df_new, 'Issue', 'Top 5 Most Common Issues', palette_name='plasma')
+    return fig
+
+# 3. Plotting top 5 issues in each product category
+def plot_top_5_issues_in_product(df_new):
+    # Step 1: Group data by 'Product' and 'Issue', then count occurrences
+    grouped_data = df_new.groupby(['Product', 'Issue']).size().reset_index(name='Count')
+
+    # Calculate total issues per product for ordering
+    total_issues_per_product = grouped_data.groupby('Product')['Count'].sum().reset_index(name='TotalIssues')
+
+    # Sort products by total issues in descending order
+    sorted_products = total_issues_per_product.sort_values('TotalIssues', ascending=False)
+
+    # Step 2: Get top 5 issues for each product sorted by 'Count' in descending order
+    top_issues_per_product = (grouped_data.groupby('Product', as_index=False)
+                            .apply(lambda x: x.nlargest(5, 'Count'))
+                            .reset_index(drop=True))
+
+    # Merge to get the order column (TotalIssues) in top_issues_per_product for sorting
+    top_issues_per_product = top_issues_per_product.merge(sorted_products, on='Product')
+
+    # Sort top_issues_per_product DataFrame based on TotalIssues column to ensure the plot respects this order
+    top_issues_per_product = top_issues_per_product.sort_values(by=['TotalIssues', 'Count'], ascending=[False, False])
+
+    # Step 3: Create a vertical stacked bar chart
+    fig = px.bar(top_issues_per_product, x='Product', y='Count', color='Issue',
+                labels={'Count': 'Number of Complaints'}, 
+                category_orders={'Product': sorted_products['Product'].tolist()}) # Explicitly set the order of products
+
+    # Update layout to remove legend and adjust dimensions for clarity
+    fig.update_layout(showlegend=False, width=900, height=600)
+    return fig
+
+# 4.Companies with the Most Complaints in 2023
+def plot_top_10_companies_complaints(df_new):
+    # Filter data for the year 2023
+    df_2023 = df_new[df_new['Date received'].dt.year == 2023]
+
+    # Group data by company name and count the number of complaints for each company
+    company_complaint_counts = df_2023['Company'].value_counts()
+
+    top_n = 10
+    # Ensure the companies are sorted in ascending order for correct plotting
+    top_companies = company_complaint_counts.head(top_n).sort_values(ascending=True)
+
+    # Create a horizontal bar chart using Plotly Express with a nicer color scale
+    fig = px.bar(
+        x=top_companies.values,
+        y=top_companies.index,
+        orientation='h',
+        color=top_companies.values, # This assigns a color based on the value
+        color_continuous_scale=[(0.0, "green"),
+                                (0.05, "yellow"),
+                                (1.0, "red")], # This is an example of a nice color scale
+        labels={'x': 'Number of Complaints', 'y': 'Company'}
+    )
+
+    fig.update_layout(
+        xaxis=dict(
+            title='Number of Complaints',
+        ),
+        yaxis=dict(
+            tickfont=dict(size=10),
+        ),
+        height=500,
+        width=800,
+    )
+
+    # To display a color bar, showing the mapping of colors to values
+    fig.update_layout(coloraxis_showscale=False)
+    return fig
+
+# 5. Top 10 States with the Most Complaints
+def plot_top_10_states_most_complaints(df_new):
+    # Assuming df_new is your DataFrame and 'State' contains the abbreviations
+    # Map state abbreviations to full names
+    df_new['State Name'] = df_new['State'].map(state_mapping)
+
+    # Calculate complaint counts by state
+    state_complaint_counts = df_new['State Name'].value_counts()
+
+    # Get top 10 states with the most complaint counts
+    top_n = 10
+    top_states = state_complaint_counts.head(top_n)
+
+    # Create a horizontal bar chart using Plotly Express with a nice color scale
+    fig = px.bar(
+        x=top_states.values,
+        y=top_states.index,
+        orientation='h',
+        color=top_states.values,  # Assign color based on values
+        color_continuous_scale='Turbo',  # A nice color scale
+        labels={'x': 'Number of Complaints', 'y': 'State'},
+        category_orders={'y': top_states.index.tolist()}
+    )
+
+    fig.update_layout(
+        yaxis=dict(
+            tickfont=dict(size=10),
+        ),
+        xaxis=dict(
+            tickangle=0,
+        ),
+        height=500,
+        width=900,
+    )
+
+    # To display a color bar, showing the mapping of colors to values
+    fig.update_layout(coloraxis_showscale=False)
+    return fig
+
+# 6. Top 10 States with the Least Complaints
+def plot_top_10_states_least_complaints(df_new):
+    # Map state abbreviations to full names
+    df_new['State Name'] = df_new['State'].map(state_mapping)
+
+    # Calculate complaint counts by state
+    state_complaint_counts = df_new['State Name'].value_counts()
+
+    # Get top 10 states with the most complaint counts
+    top_n = 10
+    top_states = state_complaint_counts.tail(top_n)
+
+    # Create a horizontal bar chart using Plotly Express with a nice color scale
+    fig = px.bar(
+        x=top_states.values,
+        y=top_states.index,
+        orientation='h',
+        color=top_states.values,  # Assign color based on values
+        color_continuous_scale='Temps',  # A nice color scale
+        labels={'x': 'Number of Complaints', 'y': 'State'},
+        category_orders={'x': top_states.index.tolist()}
+    )
+
+    fig.update_layout(
+        yaxis=dict(
+            tickfont=dict(size=10),
+        ),
+        xaxis=dict(
+            tickangle=0,
+        ),
+        height=500,
+        width=900,
+    )
+
+    # To display a color bar, showing the mapping of colors to values
+    fig.update_layout(coloraxis_showscale=False)
+
+    return fig
+
+# 7. Number of Complaints by Year
+def complaints_by_year(df_new):
+    monthly_complaints = df_new.copy()
+    monthly_complaints = monthly_complaints[monthly_complaints['Date received'].dt.year != 2024]
+
+    monthly_complaints['MonthYear'] = monthly_complaints['Date received'].dt.to_period('M').astype(str)
+    monthly_complaints = monthly_complaints.groupby('MonthYear').size().reset_index(name = "NumComplaints")
+
+
+    fig = px.line(monthly_complaints, x='MonthYear', y='NumComplaints',
+                labels={'MonthYear': 'Year', 'NumComplaints': 'Number of Complaints'})
+
+    fig.update_layout(
+            width=900,
+            height=400
+        )
+    return fig
+
+# 8. Number of Complaints by State
+def complaints_across_states(df_new):
+    df_2023 = df_new[df_new['Date received'].dt.year == 2023]
+
+    state_complaints = df_2023.groupby('State').size().reset_index(name='Num_complaints')
+    state_complaints['Full_state_name'] = state_complaints['State'].apply(lambda x : state_mapping[x] if x in state_mapping else x)
+
+    fig = px.choropleth(state_complaints,
+                        locations='State',
+                        locationmode='USA-states',
+                        color='Num_complaints',
+                        color_continuous_scale='Inferno',
+                        scope="usa",
+                        hover_name='Full_state_name')
+    fig.add_scattergeo(
+        locations=state_complaints['State'],    ###codes for states,
+        locationmode='USA-states',
+        text=state_complaints['State'],
+        mode='text',
+        hoverinfo='skip',
+        textfont=dict(size = 8.5,color='white'))
+
+    fig.update_layout(
+        autosize = True,
+        geo=dict(
+            landcolor='rgb(217, 217, 217)',  
+            lakecolor='rgb(255, 255, 255)',  
+            bgcolor='rgb(255, 255, 255)' 
+        ),
+        paper_bgcolor='rgb(255, 255, 255)', 
+        margin={"r":0,"t":50,"l":0,"b":0},
+        width=1000,
+        height=400
+    )
+    return fig

requirements.txt

@@ -0,0 +1,10 @@
+matplotlib==3.8.3
+numpy==1.26.4
+pandas==2.2.2
+plotly==5.20.0
+scikit_learn==1.4.1.post1
+seaborn==0.13.2
+streamlit==1.33.0
+streamlit_option_menu==0.3.12
+torch==2.2.2
+transformers==4.39.3