Update app.py

app.py

@@ -10,11 +10,27 @@ import plotly.graph_objects as go
 # Set page configuration
 st.set_page_config(layout="wide")
 
-# Read data into dataframes
-df1 = pd.read_csv("data/reviewed_social_media_english.csv")
-df2 = pd.read_csv("data/reviewed_news_english.csv")
-df3 = pd.read_csv("data/tamil_social_media.csv")
-df4 = pd.read_csv("data/tamil_news.csv")
+
+def load_and_clean_data():
+    df1 = pd.read_csv("data/reviewed_social_media_english.csv")
+    df2 = pd.read_csv("data/reviewed_news_english.csv")
+    df3 = pd.read_csv("data/tamil_social_media.csv")  
+    df4 = pd.read_csv("data/tamil_news.csv")       
+
+    # Normalize Text and Drop irrelevant data
+    frames = [df1, df2, df3, df4]
+    for frame in frames:
+        frame['Domain'].replace("MUSLIM", "Muslim", inplace=True)
+        frame.drop(frame[frame['Domain'] == 'Not relevant'].index, inplace=True)
+        frame.drop(frame[frame['Domain'] == 'None'].index, inplace=True)
+        frame.drop(frame[frame['Discrimination'] == 'None'].index, inplace=True)
+        frame.drop(frame[frame['Sentiment'] == 'None'].index, inplace=True)
+
+    # Concatenate/merge dataframes
+    df_combined = pd.concat(frames)
+    return df_combined
+
+df = load_and_clean_data()
 
 # Normalize Text
 df1['Domain'].replace("MUSLIM", "Muslim", inplace=True)
@@ -100,14 +116,22 @@ def render_dashboard():
 
     # ... [Other pages]
 
-# Sidebar Filters
-domain_filter = st.sidebar.multiselect('Select Domain', options=df['Domain'].unique(), default=df['Domain'].unique())
-channel_filter = st.sidebar.multiselect('Select Channel', options=df['Channel'].unique(), default=df['Channel'].unique())
-sentiment_filter = st.sidebar.multiselect('Select Sentiment', options=df['Sentiment'].unique(), default=df['Sentiment'].unique())
-discrimination_filter = st.sidebar.multiselect('Select Discrimination', options=df['Discrimination'].unique(), default=df['Discrimination'].unique())
-
-# Apply the filters
-df_filtered = df[df['Domain'].isin(domain_filter) & df['Channel'].isin(channel_filter) & df['Sentiment'].isin(sentiment_filter) & df['Discrimination'].isin(discrimination_filter)]
+# Define Sidebar Filters
+domain_options = df['Domain'].unique()
+channel_options = df['Channel'].unique()
+sentiment_options = df['Sentiment'].unique()
+discrimination_options = df['Discrimination'].unique()
+
+domain_filter = st.sidebar.multiselect('Select Domain', options=domain_options, default=domain_options)
+channel_filter = st.sidebar.multiselect('Select Channel', options=channel_options, default=channel_options)
+sentiment_filter = st.sidebar.multiselect('Select Sentiment', options=sentiment_options, default=sentiment_options)
+discrimination_filter = st.sidebar.multiselect('Select Discrimination', options=discrimination_options, default=discrimination_options)
+
+# Apply the filters to the dataframe
+df_filtered = df[(df['Domain'].isin(domain_filter)) & 
+                 (df['Channel'].isin(channel_filter)) & 
+                 (df['Sentiment'].isin(sentiment_filter)) & 
+                 (df['Discrimination'].isin(discrimination_filter))]
 
 # Render the dashboard with filtered data
 render_dashboard(df_filtered)