app.py

#app.py
import streamlit as st
import pandas as pd
import os
import glob
from src.data_processor import DataProcessor
from src.insights_generator import InsightsGenerator
from src.visualizations import *
from dotenv import load_dotenv
from parse_raw_data import run_parser

# Load environment variables from .env file
load_dotenv()

# --- Page Configuration ---
st.set_page_config(
    page_title="Prime Bank Analytics Dashboard",
    page_icon="🏦",
    layout="wide"
)

# --- Helper function to identify text column ---
def find_text_column(df):
    if df.empty:
        return None
    text_columns = [
        'text', 'Text', 'content', 'Content', 'message', 'Message',
        'review', 'Review', 'comment', 'Comment', 'post', 'Post',
        'review_text', 'Review Text', 'post_text', 'Post Text',
        'comment_text', 'Comment Text', 'description', 'Description'
    ]
    for col in text_columns:
        if col in df.columns:
            return col
    for col in df.columns:
        col_lower = col.lower()
        if any(keyword in col_lower for keyword in ['text', 'content', 'review', 'comment', 'post']):
            return col
    for col in df.columns:
        if df[col].dtype == 'object':
            sample = df[col].dropna().head()
            if not sample.empty:
                try:
                    if sample.astype(str).str.len().mean() > 20:
                        return col
                except:
                    continue
    return None

# --- Caching for Performance ---
# @st.cache_data  # Temporarily disabled to fix API key issue
def load_and_process_data():
    DATA_DIR = 'data/uploads'
    PERFECTED_DATA_DIR = 'perfected_data'
    run_parser()
    if not os.path.exists(DATA_DIR):
        os.makedirs(DATA_DIR)
    if not os.path.exists(PERFECTED_DATA_DIR):
        os.makedirs(PERFECTED_DATA_DIR)

    all_files = glob.glob(os.path.join(DATA_DIR, '*'))
    # --- CHANGED: Look for a CSV file for perfected data to preserve links and other metadata ---
    perfected_data_file = os.path.join(PERFECTED_DATA_DIR, 'all_posts_with_comments.csv')

    if not all_files and not os.path.exists(perfected_data_file):
        return pd.DataFrame(), pd.DataFrame(), None, pd.DataFrame()

    post_files = [f for f in all_files if 'post' in os.path.basename(f).lower() and f.endswith('.csv')]
    comment_files = [f for f in all_files if 'comment' in os.path.basename(f).lower() and f.endswith('.csv')]
    txt_files = [f for f in all_files if f.endswith('.txt')]
    other_csv_files = [f for f in all_files if f.endswith('.csv') and f not in post_files and f not in comment_files]

    def read_files(files_list, file_type):
        dfs = []
        for f in files_list:
            try:
                if f.endswith('.csv'):
                    df = pd.read_csv(f, on_bad_lines='skip')
                else:  # txt
                    with open(f, 'r', encoding='utf-8') as file:
                        df = pd.DataFrame({'text': [p.strip() for p in file.read().split('\n') if p.strip()]})
                
                text_col = find_text_column(df)
                if not text_col:
                    continue
                if text_col != 'text':
                    df = df.rename(columns={text_col: 'text'})
                
                df['source_file'] = os.path.basename(f)
                df['file_type'] = file_type
                df = df[df['text'].notna() & (df['text'].str.strip() != '')]
                if not df.empty:
                    dfs.append(df)
            except Exception as e:
                st.error(f"Error reading {os.path.basename(f)}: {e}")
        return pd.concat(dfs, ignore_index=True) if dfs else pd.DataFrame()

    # Load regular data for general insights
    raw_posts_df = read_files(post_files + other_csv_files, 'post')
    raw_comments_df = read_files(comment_files + txt_files, 'comment')

    # --- MODIFIED: Load perfected data from CSV for AI recommendations & Action Items ---
    perfected_df = pd.DataFrame()
    if os.path.exists(perfected_data_file):
        try:
            perfected_df = pd.read_csv(perfected_data_file, on_bad_lines='skip')
            perfected_df['source_file'] = os.path.basename(perfected_data_file)
            perfected_df['file_type'] = 'perfected'
            # Ensure text column is standardized
            text_col = find_text_column(perfected_df)
            if text_col and text_col != 'text':
                perfected_df = perfected_df.rename(columns={text_col: 'text'})
            perfected_df = perfected_df[perfected_df['text'].notna() & (perfected_df['text'].str.strip() != '')]
        except Exception as e:
            st.error(f"Error reading perfected data file '{perfected_data_file}': {e}")

    # --- Pass the API key from your .env file to the processors ---
    openai_key = "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"
    processor = DataProcessor(openai_api_key=openai_key)
    insight_gen = InsightsGenerator(openai_api_key=openai_key)

    # Process all data sets
    processed_posts_df = processor.process_all_data(raw_posts_df)
    processed_comments_df = processor.process_all_data(raw_comments_df)
    processed_perfected_df = processor.process_all_data(perfected_df) if not perfected_df.empty else pd.DataFrame()

    # Create combined dataframe for general dashboard (excluding perfected data)
    all_text_df = pd.concat([processed_posts_df, processed_comments_df], ignore_index=True)
    if all_text_df.empty and processed_perfected_df.empty:
        return pd.DataFrame(), pd.DataFrame(), None, pd.DataFrame()

    # Generate general insights from regular data
    insights = insight_gen.generate_all_insights(posts_df=processed_posts_df, all_text_df=all_text_df)

    # --- MODIFIED: Generate AI Recommendations ONLY from perfected data ---
    insights['ai_recommendations'] = {}
    if not processed_perfected_df.empty and 'prime_mentions' in processed_perfected_df.columns:
        prime_perfected_df = processed_perfected_df[processed_perfected_df['prime_mentions'] > 0].copy()
        if not prime_perfected_df.empty:
            insights['ai_recommendations'] = insight_gen.generate_ai_recommendations(prime_perfected_df)
        
    return processed_posts_df, all_text_df, insights, processed_perfected_df

# --- Main Application ---
st.title("🏦 Prime Bank Social Media Analytics")

# Manual API key input if not found in environment
if not os.getenv("OPENAI_API_KEY"):
    api_key = st.sidebar.text_input("🔑 Enter OpenAI API Key:", type="password", help="Key not found in environment")
    if api_key:
        os.environ["OPENAI_API_KEY"] = api_key

# --- MODIFIED: Renamed variable for clarity ---
posts_df, all_text_df, insights, processed_perfected_df = load_and_process_data()

if all_text_df.empty and processed_perfected_df.empty:
    st.error("No data files found. Please add CSV/TXT files to 'data/uploads' or 'all_posts_with_comments.csv' to 'perfected_data'.")
    st.info("The 'AI Recommendations' and 'Action Items' tabs are powered exclusively by 'perfected_data/all_posts_with_comments.csv'.")
    st.stop()

# Filter regular data for Prime Bank mentions for general insights
prime_posts_df = posts_df[posts_df['prime_mentions'] > 0].copy() if 'prime_mentions' in posts_df.columns else pd.DataFrame()
prime_all_text_df = all_text_df[all_text_df['prime_mentions'] > 0].copy() if 'prime_mentions' in all_text_df.columns else pd.DataFrame()

# --- KPI Section (uses general data from 'uploads' folder) ---
st.header("📈 Prime Bank Key Performance Indicators")
total_mentions = all_text_df['prime_mentions'].sum() if 'prime_mentions' in all_text_df.columns else 0
total_posts_with_mentions = len(prime_posts_df)
new_metrics = create_summary_metrics(all_text_df)

kpi1, kpi2, kpi3, kpi4 = st.columns(4)
kpi1.metric("Total Mentions (Posts & Comments)", f"{int(total_mentions):,}")
kpi2.metric("Posts Mentioning Prime Bank", f"{total_posts_with_mentions:,}")
kpi3.metric("Bank Sentiment Score", new_metrics['Bank Sentiment Score'], help="Positive Mentions - Negative Mentions. A positive score is good.")
kpi4.metric("Engagement-Weighted Sentiment", new_metrics['Engagement-Weighted Sentiment'], help="A combined score of sentiment polarity and virality (likes, shares, etc.). Higher is better.")
st.markdown("---")

# --- Tabbed Interface ---
tab1, tab2, tab3, tab4, tab5, tab6 = st.tabs([
    "Sentiment & Virality (Posts)",
    "Emotion & Categories (All Text)",
    "Strategic Overview",
    "🤖 AI Recommendations",
    "Action Items",
    "Full Data View"
])

# --- Tab 1 to 3 use GENERAL data from 'uploads' folder ---
with tab1:
    st.header("Sentiment & Virality Analysis (Posts Only)")
    if not prime_posts_df.empty:
        col1, col2 = st.columns([1, 2])
        with col1:
            st.subheader("Sentiment of Posts")
            st.plotly_chart(create_sentiment_pie(prime_posts_df), use_container_width=True)
        with col2:
            st.subheader("Top Viral Posts")
            viral_chart = create_viral_posts_chart(prime_posts_df)
            if viral_chart:
                st.plotly_chart(viral_chart, use_container_width=True)
            else:
                st.info("No viral score data (likes, shares, comments) found to display chart.")
    else:
        st.info("No posts mentioning Prime Bank were found in the data.")

with tab2:
    st.header("Emotion & Category Analysis (Posts & Comments)")
    if not prime_all_text_df.empty:
        col1, col2 = st.columns(2)
        with col1:
            st.subheader("Emotion Detection")
            st.plotly_chart(create_emotion_bar(prime_all_text_df), use_container_width=True)
        with col2:
            st.subheader("Post & Comment Categories")
            st.plotly_chart(create_category_donut(prime_all_text_df), use_container_width=True)
    else:
        st.info("No text mentioning Prime Bank was found in the data.")

with tab3:
    st.header("Strategic Overview")
    st.write("High-level insights into market position and geographic distribution.")
    col1, col2 = st.columns(2)
    with col1:
        bank_comp_chart = create_bank_comparison_chart(all_text_df)
        if bank_comp_chart:
            st.plotly_chart(bank_comp_chart, use_container_width=True)
        else:
            st.info("Not enough data to compare bank mentions.")
    with col2:
        geo_map = create_geolocation_map(all_text_df)
        if geo_map:
            st.plotly_chart(geo_map, use_container_width=True)
        else:
            # Message is handled inside the create_geolocation_map function
            pass

# --- Tab 4: AI Recommendations (uses PERFECTED data) ---
with tab4:
    st.header("🤖 AI-Powered Strategic Recommendations")
    st.info("These recommendations are generated exclusively from the curated data in `perfected_data/all_posts_with_comments.csv`.")
    if insights and 'ai_recommendations' in insights and insights['ai_recommendations']:
        recs = insights['ai_recommendations']
        st.subheader("For Customer Complaints")
        with st.expander("Show AI Insight on Complaints", expanded=True):
            st.markdown(f"💡 {recs.get('Complaint', 'No specific recommendation available.')}")

        st.subheader("For Customer Suggestions")
        with st.expander("Show AI Insight on Suggestions"):
            st.markdown(f"💡 {recs.get('Suggestion', 'No specific recommendation available.')}")
        
        st.subheader("For Customer Praise")
        with st.expander("Show AI Insight on Praise"):
            st.markdown(f"💡 {recs.get('Praise', 'No specific recommendation available.')}")
            
        st.subheader("For Customer Inquiries")
        with st.expander("Show AI Insight on Inquiries"):
            st.markdown(f"💡 {recs.get('Inquiry', 'No specific recommendation available.')}")
    else:
        st.warning("No AI recommendations could be generated. This requires a valid `perfected_data/all_posts_with_comments.csv` file with relevant data and a configured OpenAI API key.")

# --- Tab 5: Action Items (uses PERFECTED data) ---
# --- Tab 5: Action Items (uses PERFECTED data) ---
with tab5:
    st.header("Posts & Comments That Need Attention")
    st.info("This prioritized list is generated exclusively from the curated data in `perfected_data/all_posts_with_comments.csv`.")
    
    if not processed_perfected_df.empty and 'prime_mentions' in processed_perfected_df.columns:
        prime_perfected_df = processed_perfected_df[processed_perfected_df['prime_mentions'] > 0].copy()
        
        attention_df = prime_perfected_df[
            (prime_perfected_df['sentiment'] == 'Negative') | 
            (prime_perfected_df['category'].isin(['Complaint', 'Inquiry']))
        ].copy()

        if not attention_df.empty:
            # Calculate priority score
            attention_df['priority_score'] = (
                (attention_df['sentiment'] == 'Negative') * 2 +
                (attention_df['category'] == 'Complaint') * 1.5 +
                (attention_df['category'] == 'Inquiry') * 1
            )
            attention_df.sort_values(by=['priority_score', 'viral_score'], ascending=[False, False], inplace=True)
            
            # Prepare display columns and add clickable source link
            display_columns = ['text', 'sentiment', 'category', 'emotion', 'viral_score']
            link_col = None
            if 'link' in attention_df.columns:
                link_col = 'link'
            elif 'url' in attention_df.columns:
                link_col = 'url'

            if link_col:
                # Check if links are real or just placeholders
                attention_df['Source'] = attention_df[link_col].apply(
                    lambda url: "Customer Post" if pd.notna(url) and 'example.com' in str(url) 
                    else (f"[Open Post ↗]({url})" if pd.notna(url) and str(url).startswith('http') 
                    else "Direct Feedback")
                )
                display_columns.insert(1, 'Source')
            
            st.dataframe(attention_df[display_columns], use_container_width=True, hide_index=True)
        else:
            st.success("✅ No negative comments or inquiries found in the perfected data that require attention.")
    else:
        st.warning("No data found in `perfected_data/all_posts_with_comments.csv` to analyze for action items.")

# --- Tab 6: Data View (shows both general and perfected data separately) ---
with tab6:
    st.header("Explore the Raw and Processed Data")
    
    if not processed_perfected_df.empty:
        st.subheader("Processed Perfected Data (from `perfected_data`)")
        st.dataframe(processed_perfected_df.head(100))

    if not posts_df.empty:
        st.subheader("Processed Posts Data (from `data/uploads`)")
        st.dataframe(posts_df.head(100))

    comments_df = all_text_df[all_text_df['file_type'] == 'comment'] if 'file_type' in all_text_df.columns else pd.DataFrame()
    if not comments_df.empty:
        st.subheader("Processed Comments & Reviews Data (from `data/uploads`)")
        st.dataframe(comments_df.head(100))