Upload 5 files

app.py

@@ -0,0 +1,130 @@
+import streamlit as st
+import preprocessor, helper
+import matplotlib.pyplot as plt
+import seaborn as sns
+
+
+st.sidebar.title("Whatsapp Chat Analyzer")
+
+
+uploaded_file = st.sidebar.file_uploader("Choose a file")
+if uploaded_file is not None:
+    bytes_data = uploaded_file.getvalue()
+    data = bytes_data.decode("utf-8")
+    df = preprocessor.preprocess(data)
+    #st.dataframe(df)
+
+    # fetch unique users
+    user_list = df['user'].unique().tolist()
+    user_list.remove('group_notification')
+    user_list.sort()
+    user_list.insert(0, "Overall")
+
+    selected_user = st.sidebar.selectbox("Show analysis wrt", user_list)
+
+    if st.sidebar.button("Show Analysis"):
+        num_messages, num_words, num_media_messages, num_links = helper.fetch_stats(selected_user,df)
+
+        st.title("Top Statistics")
+        col1, col2, col3, col4 = st.columns(4)
+
+        with col1:
+            st.header("Total Messages")
+            st.title(num_messages)
+        with col2:
+            st.header("Total Words")
+            st.title(num_words)
+        with col3:
+            st.header("Media Shared")
+            st.title(num_media_messages)
+        with col4:
+            st.header("Links Shared")
+            st.title(num_links)
+
+        # monthly timeline
+        st.title("Monthly Timeline")
+        timeline = helper.monthly_timeline(selected_user, df)
+        fig, ax = plt.subplots()
+        ax.plot(timeline['time'], timeline['message'], color='green')
+        plt.xticks(rotation='vertical')
+        st.pyplot(fig)
+
+        # daily timeline
+        st.title("Daily Timeline")
+        daily_timeline = helper.daily_timeline(selected_user, df)
+        fig, ax = plt.subplots()
+        ax.plot(daily_timeline['only_date'], daily_timeline['message'], color='black')
+        plt.xticks(rotation='vertical')
+        st.pyplot(fig)
+
+        # activity map
+        st.title('Activity Map')
+        col1, col2 = st.columns(2)
+
+        with col1:
+            st.header("Most busy day")
+            busy_day = helper.week_activity_map(selected_user, df)
+            fig, ax = plt.subplots()
+            ax.bar(busy_day.index, busy_day.values, color='purple')
+            plt.xticks(rotation='vertical')
+            st.pyplot(fig)
+
+        with col2:
+            st.header("Most busy month")
+            busy_month = helper.month_activity_map(selected_user, df)
+            fig, ax = plt.subplots()
+            ax.bar(busy_month.index, busy_month.values, color='orange')
+            plt.xticks(rotation='vertical')
+            st.pyplot(fig)
+
+        st.title("Weekly Activity Map")
+        user_heatmap = helper.activity_heatmap(selected_user, df)
+        fig, ax = plt.subplots()
+        ax = sns.heatmap(user_heatmap)
+        st.pyplot(fig)
+
+        # finding the busiest users in the group(Group level)
+        if selected_user == 'Overall':
+            st.title('Most Busy Users')
+            x, user_df = helper.most_busy_users(df)
+            fig, ax = plt.subplots()
+
+            col1, col2 = st.columns(2)
+
+            with col1:
+                ax.bar(x.index, x.values, color='red')
+                plt.xticks(rotation='vertical')
+                st.pyplot(fig)
+            with col2:
+                st.dataframe(user_df)
+
+        # WordCloud
+        st.title("Wordcloud")
+        df_wc = helper.create_wordcloud(selected_user, df)
+        fig, ax = plt.subplots()
+        ax.imshow(df_wc)
+        st.pyplot(fig)
+
+        # most common words
+        most_common_df = helper.most_common_words(selected_user, df)
+
+        fig, ax = plt.subplots()
+
+        ax.barh(most_common_df[0], most_common_df[1])
+        plt.xticks(rotation='vertical')
+
+        st.title('Most commmon words')
+        st.pyplot(fig)
+
+        # emoji analysis
+        emoji_df = helper.emoji_helper(selected_user, df)
+        st.title("Emoji Analysis")
+
+        col1, col2 = st.columns(2)
+
+        with col1:
+            st.dataframe(emoji_df)
+        with col2:
+            fig, ax = plt.subplots()
+            ax.pie(emoji_df[1].head(), labels=emoji_df[0].head(), autopct="%0.2f")
+            st.pyplot(fig)

helper.py

@@ -0,0 +1,154 @@
+from urlextract import URLExtract
+from wordcloud import WordCloud
+import pandas as pd
+from collections import Counter
+import emoji
+
+extract = URLExtract()
+
+def fetch_stats(selected_user,df):
+
+    if selected_user != 'Overall':
+        df = df[df['user'] == selected_user]
+
+    # fetch the number of messages
+    num_messages = df.shape[0]
+
+    # fetch the total number of words
+    words = []
+    for message in df['message']:
+        words.extend(message.split())
+
+    # fetch number of media messages
+    num_media_messages = df[df['message'] == '<Media omitted>\n'].shape[0]
+
+    # fetch number of links shared
+    links = []
+    for message in df['message']:
+        links.extend(extract.find_urls(message))
+
+    return num_messages,len(words),num_media_messages,len(links)
+
+def most_busy_users(df):
+    x = df['user'].value_counts().head()
+    df = round((df['user'].value_counts() / df.shape[0]) * 100, 2).reset_index().rename(
+        columns={'index': 'name', 'user': 'percent'})
+    return x,df
+
+def create_wordcloud(selected_user,df):
+
+    f = open('stop_hinglish.txt', 'r')
+    stop_words = f.read()
+
+    if selected_user != 'Overall':
+        df = df[df['user'] == selected_user]
+
+    temp = df[df['user'] != 'group_notification']
+    temp = temp[temp['message'] != '<Media omitted>\n']
+
+    def remove_stop_words(message):
+        y = []
+        for word in message.lower().split():
+            if word not in stop_words:
+                y.append(word)
+        return " ".join(y)
+
+    wc = WordCloud(width=500,height=500,min_font_size=10,background_color='white')
+    temp['message'] = temp['message'].apply(remove_stop_words)
+    df_wc = wc.generate(temp['message'].str.cat(sep=" "))
+    return df_wc
+
+def most_common_words(selected_user,df):
+
+    f = open('stop_hinglish.txt','r')
+    stop_words = f.read()
+
+    if selected_user != 'Overall':
+        df = df[df['user'] == selected_user]
+
+    temp = df[df['user'] != 'group_notification']
+    temp = temp[temp['message'] != '<Media omitted>\n']
+
+    words = []
+
+    for message in temp['message']:
+        for word in message.lower().split():
+            if word not in stop_words:
+                words.append(word)
+
+    most_common_df = pd.DataFrame(Counter(words).most_common(20))
+    return most_common_df
+
+def emoji_helper(selected_user,df):
+    if selected_user != 'Overall':
+        df = df[df['user'] == selected_user]
+
+    emojis = []
+    for message in df['message']:
+        emojis.extend([c for c in message if c in emoji.EMOJI_DATA])
+
+    emoji_df = pd.DataFrame(Counter(emojis).most_common(len(Counter(emojis))))
+
+    return emoji_df
+
+def monthly_timeline(selected_user,df):
+
+    if selected_user != 'Overall':
+        df = df[df['user'] == selected_user]
+
+    timeline = df.groupby(['year', 'month_num', 'month']).count()['message'].reset_index()
+
+    time = []
+    for i in range(timeline.shape[0]):
+        time.append(timeline['month'][i] + "-" + str(timeline['year'][i]))
+
+    timeline['time'] = time
+
+    return timeline
+
+def daily_timeline(selected_user,df):
+
+    if selected_user != 'Overall':
+        df = df[df['user'] == selected_user]
+
+    daily_timeline = df.groupby('only_date').count()['message'].reset_index()
+
+    return daily_timeline
+
+def week_activity_map(selected_user,df):
+
+    if selected_user != 'Overall':
+        df = df[df['user'] == selected_user]
+
+    return df['day_name'].value_counts()
+
+def month_activity_map(selected_user,df):
+
+    if selected_user != 'Overall':
+        df = df[df['user'] == selected_user]
+
+    return df['month'].value_counts()
+
+def activity_heatmap(selected_user,df):
+
+    if selected_user != 'Overall':
+        df = df[df['user'] == selected_user]
+
+    user_heatmap = df.pivot_table(index='day_name', columns='period', values='message', aggfunc='count').fillna(0)
+
+    return user_heatmap
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

preprocessor.py

@@ -0,0 +1,51 @@
+import re
+import pandas as pd
+
+def preprocess(data):
+    pattern = '\d{1,2}/\d{1,2}/\d{2,4},\s\d{1,2}:\d{2}\s-\s'
+
+    messages = re.split(pattern, data)[1:]
+    dates = re.findall(pattern, data)
+
+    df = pd.DataFrame({'user_message': messages, 'message_date': dates})
+    # convert message_date type
+    df['message_date'] = pd.to_datetime(df['message_date'], format='%d/%m/%Y, %H:%M - ')
+
+    df.rename(columns={'message_date': 'date'}, inplace=True)
+
+    users = []
+    messages = []
+    for message in df['user_message']:
+        entry = re.split('([\w\W]+?):\s', message)
+        if entry[1:]:  # user name
+            users.append(entry[1])
+            messages.append(" ".join(entry[2:]))
+        else:
+            users.append('group_notification')
+            messages.append(entry[0])
+
+    df['user'] = users
+    df['message'] = messages
+    df.drop(columns=['user_message'], inplace=True)
+
+    df['only_date'] = df['date'].dt.date
+    df['year'] = df['date'].dt.year
+    df['month_num'] = df['date'].dt.month
+    df['month'] = df['date'].dt.month_name()
+    df['day'] = df['date'].dt.day
+    df['day_name'] = df['date'].dt.day_name()
+    df['hour'] = df['date'].dt.hour
+    df['minute'] = df['date'].dt.minute
+
+    period = []
+    for hour in df[['day_name', 'hour']]['hour']:
+        if hour == 23:
+            period.append(str(hour) + "-" + str('00'))
+        elif hour == 0:
+            period.append(str('00') + "-" + str(hour + 1))
+        else:
+            period.append(str(hour) + "-" + str(hour + 1))
+
+    df['period'] = period
+
+    return df

requirements.txt

@@ -0,0 +1,7 @@
+streamlit
+matplotlib
+seaborn
+urlextract
+wordcloud
+pandas
+emoji

stop_hinglish.txt

@@ -0,0 +1,1055 @@
+.
+..
+...
+?
+-
+--
+1
+2
+3
+4
+5
+6
+7
+8
+9
+0
+a
+aadi
+aaj
+aap
+aapne
+aata
+aati
+aaya
+aaye
+ab
+abbe
+abbey
+abe
+abhi
+able
+about
+above
+accha
+according
+accordingly
+acha
+achcha
+across
+actually
+after
+afterwards
+again
+against
+agar
+ain
+aint
+ain't
+aisa
+aise
+aisi
+alag
+all
+allow
+allows
+almost
+alone
+along
+already
+also
+although
+always
+am
+among
+amongst
+an
+and
+andar
+another
+any
+anybody
+anyhow
+anyone
+anything
+anyway
+anyways
+anywhere
+ap
+apan
+apart
+apna
+apnaa
+apne
+apni
+appear
+are
+aren
+arent
+aren't
+around
+arre
+as
+aside
+ask
+asking
+at
+aur
+avum
+aya
+aye
+baad
+baar
+bad
+bahut
+bana
+banae
+banai
+banao
+banaya
+banaye
+banayi
+banda
+bande
+bandi
+bane
+bani
+bas
+bata
+batao
+bc
+be
+became
+because
+become
+becomes
+becoming
+been
+before
+beforehand
+behind
+being
+below
+beside
+besides
+best
+better
+between
+beyond
+bhai
+bheetar
+bhi
+bhitar
+bht
+bilkul
+bohot
+bol
+bola
+bole
+boli
+bolo
+bolta
+bolte
+bolti
+both
+brief
+bro
+btw
+but
+by
+came
+can
+cannot
+cant
+can't
+cause
+causes
+certain
+certainly
+chahiye
+chaiye
+chal
+chalega
+chhaiye
+clearly
+c'mon
+com
+come
+comes
+could
+couldn
+couldnt
+couldn't
+d
+de
+dede
+dega
+degi
+dekh
+dekha
+dekhe
+dekhi
+dekho
+denge
+dhang
+di
+did
+didn
+didnt
+didn't
+dijiye
+diya
+diyaa
+diye
+diyo
+do
+does
+doesn
+doesnt
+doesn't
+doing
+done
+dono
+dont
+don't
+doosra
+doosre
+down
+downwards
+dude
+dunga
+dungi
+during
+dusra
+dusre
+dusri
+dvaara
+dvara
+dwaara
+dwara
+each
+edu
+eg
+eight
+either
+ek
+else
+elsewhere
+enough
+etc
+even
+ever
+every
+everybody
+everyone
+everything
+everywhere
+ex
+exactly
+example
+except
+far
+few
+fifth
+fir
+first
+five
+followed
+following
+follows
+for
+forth
+four
+from
+further
+furthermore
+gaya
+gaye
+gayi
+get
+gets
+getting
+ghar
+given
+gives
+go
+goes
+going
+gone
+good
+got
+gotten
+greetings
+guys
+haan
+had
+hadd
+hadn
+hadnt
+hadn't
+hai
+hain
+hamara
+hamare
+hamari
+hamne
+han
+happens
+har
+hardly
+has
+hasn
+hasnt
+hasn't
+have
+haven
+havent
+haven't
+having
+he
+hello
+help
+hence
+her
+here
+hereafter
+hereby
+herein
+here's
+hereupon
+hers
+herself
+he's
+hi
+him
+himself
+his
+hither
+hm
+hmm
+ho
+hoga
+hoge
+hogi
+hona
+honaa
+hone
+honge
+hongi
+honi
+hopefully
+hota
+hotaa
+hote
+hoti
+how
+howbeit
+however
+hoyenge
+hoyengi
+hu
+hua
+hue
+huh
+hui
+hum
+humein
+humne
+hun
+huye
+huyi
+i
+i'd
+idk
+ie
+if
+i'll
+i'm
+imo
+in
+inasmuch
+inc
+inhe
+inhi
+inho
+inka
+inkaa
+inke
+inki
+inn
+inner
+inse
+insofar
+into
+inward
+is
+ise
+isi
+iska
+iskaa
+iske
+iski
+isme
+isn
+isne
+isnt
+isn't
+iss
+isse
+issi
+isski
+it
+it'd
+it'll
+itna
+itne
+itni
+itno
+its
+it's
+itself
+ityaadi
+ityadi
+i've
+ja
+jaa
+jab
+jabh
+jaha
+jahaan
+jahan
+jaisa
+jaise
+jaisi
+jata
+jayega
+jidhar
+jin
+jinhe
+jinhi
+jinho
+jinhone
+jinka
+jinke
+jinki
+jinn
+jis
+jise
+jiska
+jiske
+jiski
+jisme
+jiss
+jisse
+jitna
+jitne
+jitni
+jo
+just
+jyaada
+jyada
+k
+ka
+kaafi
+kab
+kabhi
+kafi
+kaha
+kahaa
+kahaan
+kahan
+kahi
+kahin
+kahte
+kaisa
+kaise
+kaisi
+kal
+kam
+kar
+kara
+kare
+karega
+karegi
+karen
+karenge
+kari
+karke
+karna
+karne
+karni
+karo
+karta
+karte
+karti
+karu
+karun
+karunga
+karungi
+kaun
+kaunsa
+kayi
+kch
+ke
+keep
+keeps
+keh
+kehte
+kept
+khud
+ki
+kin
+kine
+kinhe
+kinho
+kinka
+kinke
+kinki
+kinko
+kinn
+kino
+kis
+kise
+kisi
+kiska
+kiske
+kiski
+kisko
+kisliye
+kisne
+kitna
+kitne
+kitni
+kitno
+kiya
+kiye
+know
+known
+knows
+ko
+koi
+kon
+konsa
+koyi
+krna
+krne
+kuch
+kuchch
+kuchh
+kul
+kull
+kya
+kyaa
+kyu
+kyuki
+kyun
+kyunki
+lagta
+lagte
+lagti
+last
+lately
+later
+le
+least
+lekar
+lekin
+less
+lest
+let
+let's
+li
+like
+liked
+likely
+little
+liya
+liye
+ll
+lo
+log
+logon
+lol
+look
+looking
+looks
+ltd
+lunga
+m
+maan
+maana
+maane
+maani
+maano
+magar
+mai
+main
+maine
+mainly
+mana
+mane
+mani
+mano
+many
+mat
+may
+maybe
+me
+mean
+meanwhile
+mein
+mera
+mere
+merely
+meri
+might
+mightn
+mightnt
+mightn't
+mil
+mjhe
+more
+moreover
+most
+mostly
+much
+mujhe
+must
+mustn
+mustnt
+mustn't
+my
+myself
+na
+naa
+naah
+nahi
+nahin
+nai
+name
+namely
+nd
+ne
+near
+nearly
+necessary
+neeche
+need
+needn
+neednt
+needn't
+needs
+neither
+never
+nevertheless
+new
+next
+nhi
+nine
+no
+nobody
+non
+none
+noone
+nope
+nor
+normally
+not
+nothing
+novel
+now
+nowhere
+o
+obviously
+of
+off
+often
+oh
+ok
+okay
+old
+on
+once
+one
+ones
+only
+onto
+or
+other
+others
+otherwise
+ought
+our
+ours
+ourselves
+out
+outside
+over
+overall
+own
+par
+pata
+pe
+pehla
+pehle
+pehli
+people
+per
+perhaps
+phla
+phle
+phli
+placed
+please
+plus
+poora
+poori
+provides
+pura
+puri
+q
+que
+quite
+raha
+rahaa
+rahe
+rahi
+rakh
+rakha
+rakhe
+rakhen
+rakhi
+rakho
+rather
+re
+really
+reasonably
+regarding
+regardless
+regards
+rehte
+rha
+rhaa
+rhe
+rhi
+ri
+right
+s
+sa
+saara
+saare
+saath
+sab
+sabhi
+sabse
+sahi
+said
+sakta
+saktaa
+sakte
+sakti
+same
+sang
+sara
+sath
+saw
+say
+saying
+says
+se
+second
+secondly
+see
+seeing
+seem
+seemed
+seeming
+seems
+seen
+self
+selves
+sensible
+sent
+serious
+seriously
+seven
+several
+shall
+shan
+shant
+shan't
+she
+she's
+should
+shouldn
+shouldnt
+shouldn't
+should've
+si
+sir
+sir.
+since
+six
+so
+soch
+some
+somebody
+somehow
+someone
+something
+sometime
+sometimes
+somewhat
+somewhere
+soon
+still
+sub
+such
+sup
+sure
+t
+tab
+tabh
+tak
+take
+taken
+tarah
+teen
+teeno
+teesra
+teesre
+teesri
+tell
+tends
+tera
+tere
+teri
+th
+tha
+than
+thank
+thanks
+thanx
+that
+that'll
+thats
+that's
+the
+theek
+their
+theirs
+them
+themselves
+then
+thence
+there
+thereafter
+thereby
+therefore
+therein
+theres
+there's
+thereupon
+these
+they
+they'd
+they'll
+they're
+they've
+thi
+thik
+thing
+think
+thinking
+third
+this
+tho
+thoda
+thodi
+thorough
+thoroughly
+those
+though
+thought
+three
+through
+throughout
+thru
+thus
+tjhe
+to
+together
+toh
+too
+took
+toward
+towards
+tried
+tries
+true
+truly
+try
+trying
+tu
+tujhe
+tum
+tumhara
+tumhare
+tumhari
+tune
+twice
+two
+um
+umm
+un
+under
+unhe
+unhi
+unho
+unhone
+unka
+unkaa
+unke
+unki
+unko
+unless
+unlikely
+unn
+unse
+until
+unto
+up
+upar
+upon
+us
+use
+used
+useful
+uses
+usi
+using
+uska
+uske
+usne
+uss
+usse
+ussi
+usually
+vaala
+vaale
+vaali
+vahaan
+vahan
+vahi
+vahin
+vaisa
+vaise
+vaisi
+vala
+vale
+vali
+various
+ve
+very
+via
+viz
+vo
+waala
+waale
+waali
+wagaira
+wagairah
+wagerah
+waha
+wahaan
+wahan
+wahi
+wahin
+waisa
+waise
+waisi
+wala
+wale
+wali
+want
+wants
+was
+wasn
+wasnt
+wasn't
+way
+we
+we'd
+well
+we'll
+went
+were
+we're
+weren
+werent
+weren't
+we've
+what
+whatever
+what's
+when
+whence
+whenever
+where
+whereafter
+whereas
+whereby
+wherein
+where's
+whereupon
+wherever
+whether
+which
+while
+who
+whoever
+whole
+whom
+who's
+whose
+why
+will
+willing
+with
+within
+without
+wo
+woh
+wohi
+won
+wont
+won't
+would
+wouldn
+wouldnt
+wouldn't
+y
+ya
+yadi
+yah
+yaha
+yahaan
+yahan
+yahi
+yahin
+ye
+yeah
+yeh
+yehi
+yes
+yet
+you
+you'd
+you'll
+your
+you're
+yours
+yourself
+yourselves
+you've
+yup