{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [] }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" } }, "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "jUat90KIzqcN", "outputId": "c36a743f-9168-472c-927a-cbbc22234353" }, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "[nltk_data] Downloading package punkt to /root/nltk_data...\n", "[nltk_data] Unzipping tokenizers/punkt.zip.\n", "[nltk_data] Downloading package stopwords to /root/nltk_data...\n", "[nltk_data] Unzipping corpora/stopwords.zip.\n" ] } ], "source": [ "import nltk\n", "nltk.download('punkt')\n", "nltk.download('stopwords')\n", "from nltk.corpus import stopwords\n", "from nltk.tokenize import word_tokenize\n", "import string\n", "from gensim.models import Word2Vec\n", "import numpy as np\n", "from sklearn.ensemble import RandomForestClassifier\n", "from sklearn.metrics import classification_report\n", "from sklearn.model_selection import train_test_split\n", "import pandas as pd" ] }, { "cell_type": "code", "source": [ "messages = pd.read_csv('SMSSpamCollection.txt', sep='\\t', names=[\"label\", \"message\"])\n", "\n", "def text_process(mess):\n", " nopunc = [char for char in mess if char not in string.punctuation]\n", " nopunc = ''.join(nopunc)\n", " return [word for word in nopunc.split() if word.lower() not in stopwords.words('english')]\n", "\n", "messages['cleaned_message'] = messages['message'].apply(text_process)\n" ], "metadata": { "id": "ObH5DJ0ezyZm" }, "execution_count": 2, "outputs": [] }, { "cell_type": "code", "source": [ "w2v_model = Word2Vec(sentences=messages['cleaned_message'], vector_size=100, window=5, min_count=1, workers=4)" ], "metadata": { "id": "7pfLq5vZz1TT" }, "execution_count": 3, "outputs": [] }, { "cell_type": "code", "source": [ "def get_average_word2vec(tokens_list, model, vector_size):\n", " if len(tokens_list) < 1:\n", " return np.zeros(vector_size)\n", " vectorized = [model.wv[word] for word in tokens_list if word in model.wv]\n", " return np.mean(vectorized, axis=0)\n", "\n", "vector_size = 100\n", "X = np.array([get_average_word2vec(tokens, w2v_model, vector_size) for tokens in messages['cleaned_message']])\n", "y = messages['label']" ], "metadata": { "id": "UpgXgFC_z5H7" }, "execution_count": 4, "outputs": [] }, { "cell_type": "code", "source": [ "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)" ], "metadata": { "id": "tvxljXvFz7bu" }, "execution_count": 6, "outputs": [] }, { "cell_type": "code", "source": [ "rf_clf = RandomForestClassifier(n_estimators=100, random_state=42)\n", "rf_clf.fit(X_train, y_train)" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 75 }, "id": "s0lBEM4mz-Xx", "outputId": "68c80902-76d2-45c7-b4c8-459f3bc88f93" }, "execution_count": 7, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "RandomForestClassifier(random_state=42)" ], "text/html": [ "
RandomForestClassifier(random_state=42)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" ] }, "metadata": {}, "execution_count": 7 } ] }, { "cell_type": "code", "source": [ "y_pred = rf_clf.predict(X_test)\n", "print(classification_report(y_test, y_pred))" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "m0e0Upk40Bj-", "outputId": "00103909-db03-471d-be25-0cd873080f1a" }, "execution_count": 8, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ " precision recall f1-score support\n", "\n", " ham 0.93 1.00 0.96 966\n", " spam 1.00 0.50 0.67 149\n", "\n", " accuracy 0.93 1115\n", " macro avg 0.96 0.75 0.82 1115\n", "weighted avg 0.94 0.93 0.92 1115\n", "\n" ] } ] }, { "cell_type": "code", "source": [ "import pickle" ], "metadata": { "id": "BthichUZ40u2" }, "execution_count": 9, "outputs": [] }, { "cell_type": "code", "source": [ "pickle.dump(rf_clf,open('rf.pkl','wb'))" ], "metadata": { "id": "qqs2eNbN0DjC" }, "execution_count": 10, "outputs": [] } ] }