{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\kurti\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "import nltk\n",
    "import string\n",
    "import numpy as np \n",
    "import pandas as pd\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "nltk.download(\"stopwords\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_tweet(tweet):\n",
    "    \"\"\"\n",
    "    Process tweet function.\n",
    "    Input:\n",
    "        tweet: a string containing a tweet\n",
    "    Returns:\n",
    "        tweets_clean: a list of words containing the processed tweet\n",
    "\n",
    "    *Taken from Coursera NLP Specialization Course 1, week 1 programming\n",
    "    assignment*\n",
    "    \"\"\"\n",
    "    stemmer = PorterStemmer()\n",
    "    stopwords_english = stopwords.words('english')\n",
    "    # remove stock market tickers like $GE\n",
    "    tweet = re.sub(r'\\$\\w*', '', str(tweet))\n",
    "    # remove old style retweet text \"RT\"\n",
    "    tweet = re.sub(r'^RT[\\s]+', '', str(tweet))\n",
    "    # remove hyperlinks\n",
    "    tweet = re.sub(r'https?:\\/\\/.*[\\r\\n]*', '', str(tweet))\n",
    "    # remove hashtags\n",
    "    # only removing the hash # sign from the word\n",
    "    tweet = re.sub(r'#', '', str(tweet))\n",
    "    # tokenize tweets\n",
    "    tokenizer = TweetTokenizer(preserve_case=False, strip_handles=True,\n",
    "                               reduce_len=True)\n",
    "    tweet_tokens = tokenizer.tokenize(tweet)\n",
    "\n",
    "    tweets_clean = []\n",
    "    for word in tweet_tokens:\n",
    "        if (word not in stopwords_english and  # remove stopwords\n",
    "                word not in string.punctuation):  # remove punctuation\n",
    "            # tweets_clean.append(word)\n",
    "            stem_word = stemmer.stem(word)  # stemming word\n",
    "            tweets_clean.append(stem_word)\n",
    "\n",
    "    return \" \".join(tweets_clean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>all_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3796</td>\n",
       "      <td>new weapon caus un-imagin destruct destructionnon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3185</td>\n",
       "      <td>f @ing thing gishwh got soak delug go pad tamp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7769</td>\n",
       "      <td>dt rt ‰ ûïthe col polic catch pickpocket liver...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>191</td>\n",
       "      <td>aftershock back school kick great want thank e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9810</td>\n",
       "      <td>respons trauma children addict develop defens ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>7934</td>\n",
       "      <td>look like got caught rainstorm amaz disgust ti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2538</td>\n",
       "      <td>favorit ladi came volunt meet hope join youth ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2611</td>\n",
       "      <td>ux fail emv peopl want insert remov quickli li...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9756</td>\n",
       "      <td>can't find ariana grand shirt fuck tragedytrag...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>6254</td>\n",
       "      <td>murder stori america ‰ ûª first hijack</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id                                           all_text\n",
       "0  3796  new weapon caus un-imagin destruct destructionnon\n",
       "1  3185  f @ing thing gishwh got soak delug go pad tamp...\n",
       "2  7769  dt rt ‰ ûïthe col polic catch pickpocket liver...\n",
       "3   191  aftershock back school kick great want thank e...\n",
       "4  9810  respons trauma children addict develop defens ...\n",
       "5  7934  look like got caught rainstorm amaz disgust ti...\n",
       "6  2538  favorit ladi came volunt meet hope join youth ...\n",
       "7  2611  ux fail emv peopl want insert remov quickli li...\n",
       "8  9756  can't find ariana grand shirt fuck tragedytrag...\n",
       "9  6254             murder stori america ‰ ûª first hijack"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read train data\n",
    "df = pd.read_csv(\"../inputs/train.csv\")\n",
    "# shuffle data\n",
    "df = df.sample(frac=1, random_state=42).reset_index(drop=True)\n",
    "# create new column \"all_text\"\n",
    "df[\"all_text\"] = df[\"text\"] + df[\"keyword\"].fillna(\"none\") + df[\"location\"].fillna(\"none\")\n",
    "# split into features and labels\n",
    "X = df.drop([\"text\", \"keyword\", \"location\", \"target\"], axis=1)\n",
    "y = df[\"target\"]\n",
    "\n",
    "# process tweets\n",
    "X[\"all_text\"] = X[\"all_text\"].apply(process_tweet)\n",
    "X.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a dictionary mapping predictions to the tweet idx\n",
    "pred_idx_dict = {}\n",
    "# initialize kfold\n",
    "skf = StratifiedKFold(n_splits=5, shuffle=False)\n",
    "for fold, (train_idx, val_idx) in enumerate(skf.split(X=X, y=y)):\n",
    "    X_train, X_val = X.loc[train_idx, :], X.loc[val_idx, :]\n",
    "    y_train, y_val = y[train_idx], y[val_idx]\n",
    "\n",
    "    # vectorize text and store model\n",
    "    count_vect = CountVectorizer()\n",
    "    X_train_vect = count_vect.fit_transform(X_train[\"all_text\"].values)\n",
    "    X_val_vect = count_vect.transform(X_val[\"all_text\"].values)\n",
    "    \n",
    "    # classify predictions\n",
    "    clf = MultinomialNB()\n",
    "    clf.fit(X_train_vect, y_train)\n",
    "    y_preds = clf.predict(X_val_vect)\n",
    "    \n",
    "    # idx of tweet mapping to prediction of model\n",
    "    for idx, key  in enumerate(val_idx):\n",
    "        pred_idx_dict[key] = y_preds[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create df with actual and prediction\n",
    "error_df = X.copy()\n",
    "error_df.rename(columns={\"all_text\":\"processed_all_text\"}, inplace=True)\n",
    "error_df[\"all_text\"] = df[df[\"id\"] == error_df[\"id\"].values][\"all_text\"]\n",
    "error_df[\"actual\"] = y.copy()\n",
    "error_df[\"predictions\"] = pred_idx_dict.values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>processed_all_text</th>\n",
       "      <th>all_text</th>\n",
       "      <th>actual</th>\n",
       "      <th>predictions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3796</td>\n",
       "      <td>new weapon caus un-imagin destruct destructionnon</td>\n",
       "      <td>So you have a new weapon that can cause un-ima...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3185</td>\n",
       "      <td>f @ing thing gishwh got soak delug go pad tamp...</td>\n",
       "      <td>The f$&amp;amp;@ing things I do for #GISHWHES Just...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7769</td>\n",
       "      <td>dt rt ‰ ûïthe col polic catch pickpocket liver...</td>\n",
       "      <td>DT @georgegalloway: RT @Galloway4Mayor: ‰ÛÏThe...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>191</td>\n",
       "      <td>aftershock back school kick great want thank e...</td>\n",
       "      <td>Aftershock back to school kick off was great. ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9810</td>\n",
       "      <td>respons trauma children addict develop defens ...</td>\n",
       "      <td>in response to trauma Children of Addicts deve...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7608</th>\n",
       "      <td>7470</td>\n",
       "      <td>mani obliter server alway like play :D obliter...</td>\n",
       "      <td>@Eganator2000 There aren't many Obliteration s...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7609</th>\n",
       "      <td>7691</td>\n",
       "      <td>panic attack bc enough money drug alcohol want...</td>\n",
       "      <td>just had a panic attack bc I don't have enough...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7610</th>\n",
       "      <td>1242</td>\n",
       "      <td>omron hem 712c automat blood pressur monitor s...</td>\n",
       "      <td>Omron HEM-712C Automatic Blood Pressure Monito...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7611</th>\n",
       "      <td>10862</td>\n",
       "      <td>offici say quarantin place alabama home possib...</td>\n",
       "      <td>Officials say a quarantine is in place at an A...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7612</th>\n",
       "      <td>10409</td>\n",
       "      <td>move england five year ago today whirlwind time</td>\n",
       "      <td>I moved to England five years ago today. What ...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7613 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id                                 processed_all_text  \\\n",
       "0      3796  new weapon caus un-imagin destruct destructionnon   \n",
       "1      3185  f @ing thing gishwh got soak delug go pad tamp...   \n",
       "2      7769  dt rt ‰ ûïthe col polic catch pickpocket liver...   \n",
       "3       191  aftershock back school kick great want thank e...   \n",
       "4      9810  respons trauma children addict develop defens ...   \n",
       "...     ...                                                ...   \n",
       "7608   7470  mani obliter server alway like play :D obliter...   \n",
       "7609   7691  panic attack bc enough money drug alcohol want...   \n",
       "7610   1242  omron hem 712c automat blood pressur monitor s...   \n",
       "7611  10862  offici say quarantin place alabama home possib...   \n",
       "7612  10409    move england five year ago today whirlwind time   \n",
       "\n",
       "                                               all_text  actual  predictions  \n",
       "0     So you have a new weapon that can cause un-ima...       1            0  \n",
       "1     The f$&amp;@ing things I do for #GISHWHES Just...       0            0  \n",
       "2     DT @georgegalloway: RT @Galloway4Mayor: ‰ÛÏThe...       1            0  \n",
       "3     Aftershock back to school kick off was great. ...       0            0  \n",
       "4     in response to trauma Children of Addicts deve...       0            1  \n",
       "...                                                 ...     ...          ...  \n",
       "7608  @Eganator2000 There aren't many Obliteration s...       0            0  \n",
       "7609  just had a panic attack bc I don't have enough...       0            0  \n",
       "7610  Omron HEM-712C Automatic Blood Pressure Monito...       0            1  \n",
       "7611  Officials say a quarantine is in place at an A...       1            1  \n",
       "7612  I moved to England five years ago today. What ...       1            1  \n",
       "\n",
       "[7613 rows x 5 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "error_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>processed_all_text</th>\n",
       "      <th>all_text</th>\n",
       "      <th>actual</th>\n",
       "      <th>predictions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>1061</td>\n",
       "      <td>ye i'm bleed heart liberal.bleedingl oak tx</td>\n",
       "      <td>@KatRamsland Yes I'm a bleeding heart liberal....</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>518</th>\n",
       "      <td>8946</td>\n",
       "      <td>storm came . . fuck coolstormnon</td>\n",
       "      <td>So this storm just came out of no where. .fuck...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3161</th>\n",
       "      <td>143</td>\n",
       "      <td>car even week got fuck car accid .. mf can't f...</td>\n",
       "      <td>only had a car for not even a week and got in ...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6624</th>\n",
       "      <td>9044</td>\n",
       "      <td>spacex founder musk structur failur took falcon 9</td>\n",
       "      <td>SpaceX Founder Musk: Structural Failure Took D...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>881</th>\n",
       "      <td>1458</td>\n",
       "      <td>anoth one anoth one still ain't done shit one ...</td>\n",
       "      <td>'I did another one I did another one. You stil...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4314</th>\n",
       "      <td>10364</td>\n",
       "      <td>router one latest ddo attack weapon</td>\n",
       "      <td>Your Router is One of the Latest DDoS Attack W...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5399</th>\n",
       "      <td>6188</td>\n",
       "      <td>gov brown allow parol 1976 chowchilla school b...</td>\n",
       "      <td>Gov. Brown allows parole for 1976 Chowchilla s...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4266</th>\n",
       "      <td>4911</td>\n",
       "      <td>chick masturb guy get explod face</td>\n",
       "      <td>Chick masturbates a guy until she gets explode...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3959</th>\n",
       "      <td>2112</td>\n",
       "      <td>borrow concern possibl interest rate rise coul...</td>\n",
       "      <td>#Borrowers concerned at possible #interest rat...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6445</th>\n",
       "      <td>7926</td>\n",
       "      <td>stuck rainstorm stay toward middl road street ...</td>\n",
       "      <td>Stuck in a rainstorm? Stay toward the middle o...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id                                 processed_all_text  \\\n",
       "149    1061        ye i'm bleed heart liberal.bleedingl oak tx   \n",
       "518    8946                   storm came . . fuck coolstormnon   \n",
       "3161    143  car even week got fuck car accid .. mf can't f...   \n",
       "6624   9044  spacex founder musk structur failur took falcon 9   \n",
       "881    1458  anoth one anoth one still ain't done shit one ...   \n",
       "4314  10364                router one latest ddo attack weapon   \n",
       "5399   6188  gov brown allow parol 1976 chowchilla school b...   \n",
       "4266   4911                  chick masturb guy get explod face   \n",
       "3959   2112  borrow concern possibl interest rate rise coul...   \n",
       "6445   7926  stuck rainstorm stay toward middl road street ...   \n",
       "\n",
       "                                               all_text  actual  predictions  \n",
       "149   @KatRamsland Yes I'm a bleeding heart liberal....       1            0  \n",
       "518   So this storm just came out of no where. .fuck...       1            0  \n",
       "3161  only had a car for not even a week and got in ...       1            0  \n",
       "6624  SpaceX Founder Musk: Structural Failure Took D...       1            0  \n",
       "881   'I did another one I did another one. You stil...       1            0  \n",
       "4314  Your Router is One of the Latest DDoS Attack W...       0            1  \n",
       "5399  Gov. Brown allows parole for 1976 Chowchilla s...       0            1  \n",
       "4266  Chick masturbates a guy until she gets explode...       1            0  \n",
       "3959  #Borrowers concerned at possible #interest rat...       0            1  \n",
       "6445  Stuck in a rainstorm? Stay toward the middle o...       0            1  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# store only the misclassified instances\n",
    "misclassified_df = error_df[error_df[\"actual\"].values != error_df[\"predictions\"]]\n",
    "# keep only 100 of the misclassfied instances\n",
    "misclassified_100 = misclassified_df.sample(n=100, random_state=42)\n",
    "misclassified_100.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "misclassified_100.to_csv(\"misclassified_data.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}