eda.py

import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
from wordcloud import WordCloud
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from collections import Counter
import nltk
import streamlit as st

nltk.download('stopwords')
nltk.download('punkt')

data = pd.read_csv("Spam_SMS.csv")

data['Message_Length'] = data['Message'].apply(len)
data['Word_Count'] = data['Message'].apply(lambda x: len(x.split()))
data['Unique_Word_Count'] = data['Message'].apply(lambda x: len(set(x.split())))

def preprocess_text(text):
    text = text.lower()
    tokens = word_tokenize(text)
    stop_words = set(stopwords.words('english'))
    tokens = [word for word in tokens if word.isalnum() and word not in stop_words]
    return tokens

def run():
    st.title("Exploratory Data Analysis - Spam Message Detection Model")
    st.write("---")
    st.image('ham_or_spam.jpg')

    st.write("### Data Preview")
    st.write(data.head())

    class_counts = data['Class'].value_counts()
    st.write("### Class Distribution")
    st.write(class_counts)

    fig = px.pie(values=class_counts.values, names=class_counts.index, title="Class Distribution")
    st.plotly_chart(fig)
    st.write('The data is imbalanced, with 4,827 (86.6%) messages classified as not spam and only 747 (13.4%) classified as spam. However, because it is not highly imbalanced (more than 1%), no additional data balancing methods need to be applied')

    avg_message_length = data.groupby('Class')['Message_Length'].mean()
    avg_word_count = data.groupby('Class')['Word_Count'].mean()
    avg_unique_word_count = data.groupby('Class')['Unique_Word_Count'].mean()

    fig = go.Figure(data=[
        go.Bar(name='Ham', x=['ham'], y=[avg_message_length['ham']], marker_color='blue'),
        go.Bar(name='Spam', x=['spam'], y=[avg_message_length['spam']], marker_color='red')
    ])
    fig.update_layout(title='Average Message Length by Class', xaxis_title='Class', yaxis_title='Average Message Length (characters)')
    st.plotly_chart(fig)

    fig = go.Figure(data=[
        go.Bar(name='Ham', x=['ham'], y=[avg_word_count['ham']], marker_color='blue'),
        go.Bar(name='Spam', x=['spam'], y=[avg_word_count['spam']], marker_color='red')
    ])
    fig.update_layout(title='Average Word Count by Class', xaxis_title='Class', yaxis_title='Average Word Count')
    st.plotly_chart(fig)

    fig = go.Figure(data=[
        go.Bar(name='Ham', x=['ham'], y=[avg_unique_word_count['ham']], marker_color='blue'),
        go.Bar(name='Spam', x=['spam'], y=[avg_unique_word_count['spam']], marker_color='red')
    ])
    fig.update_layout(title='Average Unique Word Count by Class', xaxis_title='Class', yaxis_title='Average Unique Word Count')
    st.plotly_chart(fig)

    st.write("### Message Distribution of Spam and Ham Messages")
    st.write("On average, spam messages tend to be longer and contain more words, including a larger variety of unique words, compared to ham messages. This pattern suggests that spam messages often include additional keywords to grab attention or convey offers.")

    for message_class in ['spam', 'ham']:
        all_words = []
        for message in data[data['Class'] == message_class]['Message']:
            all_words.extend(preprocess_text(message))
        
        word_counts = Counter(all_words)
        
        wordcloud = WordCloud(width=800, height=400, background_color='white', max_words=20).generate_from_frequencies(word_counts)
        
        fig = px.imshow(wordcloud, title=f"{message_class.capitalize()} Message Word Cloud")
        fig.update_xaxes(visible=False)
        fig.update_yaxes(visible=False)
        st.plotly_chart(fig)

    st.write("### Most Common Words Found in Each Spam and Ham Messages")
    st.write("The word clouds show common words used in each message class. Spam messages often contain keywords that prompt immediate action or offer something appealing, while ham messages use more common conversational words without a distinct pattern.")

if __name__ == '__main__':
    run()