updated word cloud

app.py

@@ -8,7 +8,6 @@ import gradio as gr
 import pandas as pd
 import numpy as np
 import torch
-import re
 import sqlite3
 import json
 import logging
@@ -21,15 +20,12 @@ from io import StringIO
 from datetime import datetime, timezone
 from logging.handlers import RotatingFileHandler
 
-# --- APIs and Web Scraping ---
-from googleapiclient.discovery import build
-from googleapiclient.errors import HttpError
-from GoogleNews import GoogleNews
-from urllib.error import HTTPError
+# --- NLP & Machine Learning ---
+# BanglaBERT tokenizer removed for simplicity
 import dateparser
 
 # --- NLP & Machine Learning ---
-from transformers import pipeline, BitsAndBytesConfig
+from transformers import pipeline, BitsAndBytesConfig, AutoTokenizer
 from sentence_transformers import SentenceTransformer
 from huggingface_hub.utils import HfHubHTTPError
 
@@ -54,8 +50,8 @@ logger.info("Application starting up.")
 
 # --- APPLICATION CONFIGURATION ---
 APP_TITLE = "Social Perception Analyzer"
-APP_TAGLINE = "Prepared for the Policymakers of Bangladesh Nationalist Party (BNP)"
-APP_FOOTER = "Developed by CDSR"
+APP_TAGLINE = "A flatform for understanding Netizens dynamics"
+APP_FOOTER = "Developed by Arjon"
 
 # --- FONT CONFIGURATION ---
 FONT_PATH = 'NotoSansBengali-Regular.ttf'
@@ -69,6 +65,30 @@ except OSError:
 
 # ==============================================================================
 # CORE HELPER FUNCTIONS
+def clean_bengali_text(text):
+    # Remove non-Bengali characters except spaces and underscores (for joined phrases)
+    # Preserve word shapes by not removing valid combining marks
+    cleaned = re.sub(r'[^\u0980-\u09FF_\s]', '', str(text))
+    # Remove extra spaces
+    cleaned = re.sub(r'\s+', ' ', cleaned).strip()
+    return cleaned
+NOTEBOOK_STOPWORDS = set([
+    'এবং', 'ও', 'বা', 'কিংবা', 'অথবা', 'কিন্তু', 'এর', 'এ', 'এই', 'সেই', 'ওই', 'এক', 'জন্য',
+    'আমার', 'তোমার', 'তার', 'আমাদের', 'তাদের', 'সে', 'তিনি', 'আমি', 'তুমি', 'যে', 'যায়', 'হয়',
+    'হবে', 'ছিল', 'আছে', 'নেই', 'এটা', 'ওটা', 'সেটা', 'করে', 'করতে', 'করেছে', 'করছেন', 'থেকে',
+    'সাথে', 'মধ্যে', 'উপরে', 'নিচে', 'পরে', 'আগে', 'শুধু', 'খুব', 'অনেক', 'আরও', 'হিসাবে', 'তাহলে',
+    'হলে', 'তাই', 'সুতরাং', 'কারণে', 'একটি', 'হয়ে', 'হয়েছিল', 'হচ্ছে', 'হয়েছে', 'না', 'হ্যাঁ', 'কি',
+    'কী', 'কে', 'কোন', 'গুলো', 'কিছু', 'বলেন', 'বললেন', 'বলল', 'আর', 'ভাই', 'হোক', 'চাই', 'বাদ',
+    'দিতে', 'দিয়ে', 'দিলেন', 'দেন', 'যাবে', 'যাক', 'পারা', 'পারে', 'করা', 'করি', 'করার', 'করছে',
+    'করবে', 'সব', 'এখন', 'যদি', 'কেন', 'কবে', 'কেমন', 'ইনশাআল্লাহ', 'আপনি', 'আপনার', 'আপনারা', 'আমরা'
+])
+COMBINED_STOPWORDS = set(BANGLA_STOP_WORDS) | NOTEBOOK_STOPWORDS
+PHRASES_TO_JOIN = {
+    "তারেক রহমান": "তারেক_রহমান",
+    "খালেদা জিয়া": "খালেদা_জিয়া",
+    "বিএনপি জিন্দাবাদ": "বিএনপি_জিন্দাবাদ"
+    # Add more as needed
+}
 # ==============================================================================
 
 BANGLA_STOP_WORDS = [
@@ -101,34 +121,23 @@ def get_dynamic_time_agg(start_date, end_date):
 
 # ==============================================================================
 # ML MODEL MANAGEMENT
-# ==============================================================================
-
+TOKENIZER_MODEL_ID = "csebuetnlp/banglabert_large"
+TOKENIZER = None
 
-SENTIMENT_MODEL_ID = 'ahs95/banglabert-sentiment-analysis'
-MODELS = {"sentiment_pipeline": None}
-
-def _load_pipeline_with_retry(task, model_id, retries=3):
-    logger.info(f"Initializing {task} pipeline for model: {model_id}")
-    for attempt in range(retries):
+def get_bangla_tokenizer():
+    global TOKENIZER
+    if TOKENIZER is None:
         try:
-            device = 0 if torch.cuda.is_available() else -1
-            if device == -1: gr.Warning(f"{model_id} will run on CPU and may be very slow.")
-            pipe = pipeline(task, model=model_id, device=device)
-            logger.info(f"Pipeline '{task}' loaded successfully.")
-            return pipe
-        except (HfHubHTTPError, requests.exceptions.ConnectionError) as e:
-            logger.warning(f"Network error on loading {model_id} (Attempt {attempt + 1}/{retries}): {e}")
-            if attempt < retries - 1: time.sleep(5)
-            else: raise gr.Error(f"Failed to download model '{model_id}' after {retries} attempts. Check network.")
+            TOKENIZER = AutoTokenizer.from_pretrained(TOKENIZER_MODEL_ID)
+            logger.info("BanglaBERT tokenizer loaded successfully.")
         except Exception as e:
-            logger.error(f"An unexpected error occurred while loading {model_id}: {e}")
-            raise gr.Error(f"Could not initialize model '{model_id}'. Error: {e}")
-    return None
+            logger.error(f"Failed to load BanglaBERT tokenizer: {e}")
+            TOKENIZER = None
+    return TOKENIZER
+# ==============================================================================
 
-def get_sentiment_pipeline():
-    if MODELS["sentiment_pipeline"] is None:
-        MODELS["sentiment_pipeline"] = _load_pipeline_with_retry("sentiment-analysis", SENTIMENT_MODEL_ID)
-    return MODELS["sentiment_pipeline"]
+
+## Sentiment pipeline code removed for optimization
 
 # ==============================================================================
 # NEWS SCRAPER BACKEND
@@ -148,35 +157,35 @@ def run_news_scraper_pipeline(search_keywords, sites, start_date_str, end_date_s
 
     all_articles, current_dt = [], start_dt
     while current_dt <= end_dt:
-        interval_end_dt = min(current_dt + pd.Timedelta(days=interval - 1), end_dt)
-        start_str, end_str = current_dt.strftime('%Y-%m-%d'), interval_end_dt.strftime('%Y-%m-%d')
-        progress(0, desc=f"Fetching news from {start_str} to {end_str}")
-
-        site_query = f"({' OR '.join(['site:' + s.strip() for s in sites.split(',') if s.strip()])})" if sites else ""
-        final_query = f'"{search_keywords}" {site_query} after:{start_str} before:{end_str}'
-
-        googlenews = GoogleNews(lang='bn', region='BD')
-        googlenews.search(final_query)
-        
-        for page in range(1, max_pages + 1):
-            try:
-                results = googlenews.results()
-                if not results: break
-                all_articles.extend(results)
-                if page < max_pages:
-                    googlenews.getpage(page + 1)
-                    time.sleep(random.uniform(2, 5))
-            except HTTPError as e:
-                if e.code == 429:
-                    wait_time = random.uniform(15, 30)
-                    gr.Warning(f"Rate limited by Google News. Pausing for {wait_time:.0f} seconds.")
-                    time.sleep(wait_time)
-                else:
-                    logger.error(f"HTTP Error fetching news: {e}"); break
-            except Exception as e:
-                logger.error(f"An error occurred fetching news: {e}"); break
-        
-        current_dt += pd.Timedelta(days=interval)
+        try:
+            interval_end_dt = min(current_dt + pd.Timedelta(days=interval - 1), end_dt)
+            start_str, end_str = current_dt.strftime('%Y-%m-%d'), interval_end_dt.strftime('%Y-%m-%d')
+            progress(0, desc=f"Fetching news from {start_str} to {end_str}")
+            site_query = f"({' OR '.join(['site:' + s.strip() for s in sites.split(',') if s.strip()])})" if sites else ""
+            final_query = f'"{search_keywords}" {site_query} after:{start_str} before:{end_str}'
+            googlenews = GoogleNews(lang='bn', region='BD')
+            googlenews.search(final_query)
+            for page in range(1, max_pages + 1):
+                try:
+                    results = googlenews.results()
+                    if not results: break
+                    all_articles.extend(results)
+                    if page < max_pages:
+                        googlenews.getpage(page + 1)
+                        time.sleep(0.5) # Reduced sleep for performance
+                except HTTPError as e:
+                    if e.code == 429:
+                        wait_time = 5 # Reduced wait for optimization
+                        gr.Warning(f"Rate limited by Google News. Pausing for {wait_time:.0f} seconds.")
+                        time.sleep(wait_time)
+                    else:
+                        logger.error(f"HTTP Error fetching news: {e}"); break
+                except Exception as e:
+                    logger.error(f"An error occurred fetching news: {e}"); break
+            current_dt += pd.Timedelta(days=interval)
+        except Exception as e:
+            logger.error(f"Error in news scraping loop: {e}")
+            break
 
     if not all_articles: return pd.DataFrame(), pd.DataFrame()
     
@@ -185,8 +194,40 @@ def run_news_scraper_pipeline(search_keywords, sites, start_date_str, end_date_s
     df.dropna(subset=['published_date', 'title'], inplace=True)
 
     if filter_keys and filter_keys.strip():
-        keywords = [k.strip().lower() for k in filter_keys.split(',')]
-        mask = df.apply(lambda row: any(key in str(row['title']).lower() or key in str(row['desc']).lower() for key in keywords), axis=1)
+        # Advanced filtering logic: supports AND, OR, NOT, and phrase search
+        def parse_query(query):
+            # Simple parser for AND, OR, NOT, and phrase queries
+            query = query.lower()
+            tokens = re.findall(r'"[^"]+"|\S+', query)
+            expr = []
+            for token in tokens:
+                if token == 'and': expr.append('&')
+                elif token == 'or': expr.append('|')
+                elif token == 'not': expr.append('!')
+                else:
+                    if token.startswith('"') and token.endswith('"'):
+                        expr.append(f'"{token[1:-1]}"')
+                    else:
+                        expr.append(f'"{token}"')
+            return ' '.join(expr)
+
+        def match_complex_query(text, query):
+            # Evaluate the parsed query against the text
+            text = text.lower()
+            expr = parse_query(query)
+            # Replace quoted terms with their presence in text
+            def term_eval(term):
+                term = term.strip('"')
+                return term in text
+            # Replace operators with Python equivalents
+            expr = re.sub(r'"([^"]+)"', lambda m: str(term_eval(m.group(0))), expr)
+            expr = expr.replace('&', ' and ').replace('|', ' or ').replace('!', ' not ')
+            try:
+                return eval(expr)
+            except Exception:
+                return False
+
+        mask = df.apply(lambda row: match_complex_query(str(row['title']) + ' ' + str(row['desc']), filter_keys), axis=1)
         df = df[mask]
 
     return df, df[['published_date', 'title', 'media', 'desc', 'link']].sort_values(by='published_date', ascending=False)
@@ -235,7 +276,8 @@ def _scrape_single_video_comments(youtube_service, video_id, max_comments):
     return comments_list
 
 def run_youtube_analysis_pipeline(api_key, query, max_videos_for_stats, num_videos_for_comments, max_comments_per_video, published_after, progress=gr.Progress()):
-    if not api_key: raise gr.Error("YouTube API Key is required.")
+    # Use integrated API key for seamless experience
+    api_key = "AIzaSyB_f3uROqZfwBWsc_sDEV63WmUHBgvGGqw"
     if not query: raise gr.Error("Search Keywords are required.")
     try:
         youtube = build('youtube', 'v3', developerKey=api_key)
@@ -307,22 +349,7 @@ def set_plot_style():
     plt.rcParams['figure.dpi'] = 100
 
 def run_sentiment_analysis(df: pd.DataFrame, text_column: str, progress=gr.Progress()):
-    if text_column not in df.columns: return df
-    sentiment_pipeline = get_sentiment_pipeline()
-    if not sentiment_pipeline:
-        gr.Warning("Sentiment model failed to load. Skipping analysis.")
-        return df
-
-    texts = df[text_column].dropna().tolist()
-    if not texts: return df
-
-    progress(0, desc="Running sentiment analysis...")
-    results = sentiment_pipeline(texts, batch_size=32)
-
-    text_to_sentiment = {text: result for text, result in zip(texts, results)}
-    df['sentiment_label'] = df[text_column].map(lambda x: text_to_sentiment.get(x, {}).get('label'))
-    df['sentiment_score'] = df[text_column].map(lambda x: text_to_sentiment.get(x, {}).get('score'))
-    logger.info("Sentiment analysis complete.")
+    # Sentiment analysis removed
     return df
 
 def generate_scraper_dashboard(df: pd.DataFrame):
@@ -343,11 +370,41 @@ def generate_scraper_dashboard(df: pd.DataFrame):
         ax.set_yticklabels(media_counts.index, fontproperties=BANGLA_FONT); ax.set_xlabel("Article Count"); plt.tight_layout()
 
     text = " ".join(title for title in df['title'].astype(str))
+    text = clean_bengali_text(text)
+    for phrase, joined in PHRASES_TO_JOIN.items():
+        text = text.replace(phrase, joined)
     fig_wc = None
     try:
-        wc = WordCloud(font_path=FONT_PATH, width=800, height=400, background_color='white', stopwords=BANGLA_STOP_WORDS, collocations=False).generate(text)
-        fig_wc, ax = plt.subplots(figsize=(10, 5)); ax.imshow(wc, interpolation='bilinear'); ax.axis("off")
-    except Exception as e: logger.error(f"WordCloud failed: {e}")
+        words = re.findall(r'[\u0980-\u09FF_]{2,}', text)
+        words = [w for w in words if w not in COMBINED_STOPWORDS]
+        words = [w for w in words if len(w) > 1]
+        words = [w for w in words if not re.search(r'[a-zA-Z]', w)]
+        from collections import Counter
+        word_freq = Counter(words)
+        min_freq = 2
+        most_common = set([w for w, _ in word_freq.most_common(3)])
+        filtered_words = [w for w in words if word_freq[w] >= min_freq and w not in most_common]
+        wc_text = " ".join(filtered_words)
+        wc = WordCloud(
+            font_path=FONT_PATH,
+            width=1600,
+            height=900,
+            background_color='white',
+            stopwords=COMBINED_STOPWORDS,
+            collocations=False,
+            colormap='plasma',
+            max_words=200,
+            contour_width=2,
+            contour_color='steelblue',
+            regexp=r"[\u0980-\u09FF_]+"
+        ).generate(wc_text)
+        fig_wc, ax = plt.subplots(figsize=(15, 8))
+        ax.imshow(wc, interpolation='bilinear')
+        ax.axis("off")
+        ax.set_title("Bengali Headline Word Cloud", fontproperties=BANGLA_FONT, fontsize=22)
+        plt.tight_layout()
+    except Exception as e:
+        gr.Warning(f"WordCloud failed: {e}")
     
     return {
         kpi_total_articles: str(total_articles), kpi_unique_media: str(unique_media), kpi_date_range: date_range_str,
@@ -356,74 +413,144 @@ def generate_scraper_dashboard(df: pd.DataFrame):
     }
 
 def generate_sentiment_dashboard(df: pd.DataFrame):
-    updates = {sentiment_dashboard_tab: gr.update(visible=False)}
-    set_plot_style()
-    
-    if 'sentiment_label' in df.columns:
-        sentiment_counts = df['sentiment_label'].value_counts()
-        fig_pie, fig_media_sent = None, None
-        if not sentiment_counts.empty:
-            fig_pie, ax = plt.subplots(figsize=(6, 6)); ax.pie(sentiment_counts, labels=sentiment_counts.index, autopct='%1.1f%%', startangle=90, colors=['#66c2a5', '#fc8d62', '#8da0cb'])
-            ax.set_title("Overall Sentiment Distribution", fontproperties=BANGLA_FONT); ax.axis('equal')
-        
-        top_media = df['media'].value_counts().nlargest(10).index
-        media_sentiment = pd.crosstab(df[df['media'].isin(top_media)]['media'], df['sentiment_label'], normalize='index').mul(100)
-        if not media_sentiment.empty:
-            fig_media_sent, ax = plt.subplots(figsize=(10, 7)); media_sentiment.plot(kind='barh', stacked=True, ax=ax, colormap='viridis')
-            ax.set_title("Sentiment by Top Media Sources", fontproperties=BANGLA_FONT); ax.set_yticklabels(media_sentiment.index, fontproperties=BANGLA_FONT); plt.tight_layout()
-        
-        updates.update({sentiment_pie_plot: fig_pie, sentiment_by_media_plot: fig_media_sent, sentiment_dashboard_tab: gr.update(visible=True)})
-    return updates
+    # Sentiment dashboard removed
+    return {sentiment_dashboard_tab: gr.update(visible=False)}
 
 def generate_youtube_dashboard(videos_df, comments_df):
     set_plot_style()
     kpis = {
-        kpi_yt_videos_found: f"{len(videos_df):,}" if videos_df is not None else "0",
-        kpi_yt_views_scanned: f"{videos_df['view_count'].sum():,}" if videos_df is not None else "0",
-        kpi_yt_comments_scraped: f"{len(comments_df):,}" if comments_df is not None else "0"
+        kpi_yt_videos_found: f"{len(videos_df):,}" if videos_df is not None and not videos_df.empty else "0",
+        kpi_yt_views_scanned: f"{videos_df['view_count'].sum():,}" if videos_df is not None and not videos_df.empty and 'view_count' in videos_df.columns else "0",
+        kpi_yt_comments_scraped: f"{len(comments_df):,}" if comments_df is not None and not comments_df.empty else "0"
     }
     
-    channel_counts = videos_df['channel'].value_counts().nlargest(15).sort_values()
-    fig_channels, ax = plt.subplots(figsize=(8, 6))
-    if not channel_counts.empty:
-        channel_counts.plot(kind='barh', ax=ax, color='coral'); ax.set_title("Top 15 Channels by Video Volume", fontproperties=BANGLA_FONT); ax.set_yticklabels(channel_counts.index, fontproperties=BANGLA_FONT); plt.tight_layout()
-
-    fig_wc, fig_pie, fig_sentiment_video = None, None, None
+    fig_channels, ax = None, None
+    if videos_df is not None and not videos_df.empty and 'channel' in videos_df.columns:
+        channel_counts = videos_df['channel'].value_counts().nlargest(15).sort_values()
+        if not channel_counts.empty:
+            fig_channels, ax = plt.subplots(figsize=(8, 6))
+            channel_counts.plot(kind='barh', ax=ax, color='coral'); ax.set_title("Top 15 Channels by Video Volume", fontproperties=BANGLA_FONT); ax.set_yticklabels(channel_counts.index, fontproperties=BANGLA_FONT); plt.tight_layout()
+
+    # Rich analytics: engagement, top videos, comment activity, time series, etc.
+    fig_wc, fig_top_videos, fig_engagement, fig_comment_activity, fig_time_series = None, None, None, None, None
     if comments_df is not None and not comments_df.empty:
-        text = " ".join(comment for comment in comments_df['comment_text'].astype(str))
-        try:
-            wc = WordCloud(font_path=FONT_PATH, width=800, height=400, background_color='white', stopwords=BANGLA_STOP_WORDS, collocations=False).generate(text)
-            fig_wc, ax = plt.subplots(figsize=(10, 5)); ax.imshow(wc, interpolation='bilinear'); ax.axis("off"); ax.set_title("Most Common Words in Comments", fontproperties=BANGLA_FONT)
-        except Exception as e: logger.error(f"YouTube WordCloud failed: {e}")
-
-        if 'sentiment_label' in comments_df.columns:
-            sentiment_counts = comments_df['sentiment_label'].value_counts()
-            if not sentiment_counts.empty:
-                fig_pie, ax = plt.subplots(figsize=(6, 6)); ax.pie(sentiment_counts, labels=sentiment_counts.index, autopct='%1.1f%%', startangle=90, colors=['#66c2a5', '#fc8d62', '#8da0cb']); ax.set_title("Overall Comment Sentiment", fontproperties=BANGLA_FONT)
-            
-            top_videos_by_comment = comments_df['video_title'].value_counts().nlargest(10).index
-            video_sentiment = comments_df.groupby('video_title')['sentiment_label'].value_counts(normalize=True).unstack().mul(100).reindex(top_videos_by_comment).dropna(how='all')
-            if not video_sentiment.empty:
-                fig_sentiment_video, ax = plt.subplots(figsize=(10, 8)); video_sentiment.plot(kind='barh', stacked=True, ax=ax, colormap='viridis'); ax.set_title("Comment Sentiment by Top 10 Videos", fontproperties=BANGLA_FONT); ax.set_yticklabels(video_sentiment.index, fontproperties=BANGLA_FONT); plt.tight_layout()
+        # Top commented videos
+        fig_top_videos, ax = None, None
+        if 'video_title' in comments_df.columns:
+            top_videos = comments_df['video_title'].value_counts().nlargest(10)
+            if not top_videos.empty:
+                fig_top_videos, ax = plt.subplots(figsize=(10, 6))
+                top_videos.plot(kind='barh', ax=ax, color='dodgerblue')
+                ax.set_title("Top 10 Videos by Comment Count", fontproperties=BANGLA_FONT)
+                ax.set_xlabel("Comment Count")
+                ax.set_yticklabels(top_videos.index, fontproperties=BANGLA_FONT)
+                plt.tight_layout()
+                plt.close(fig_top_videos)
+
+        # Engagement rate per video
+        fig_engagement, ax = None, None
+        if 'video_id' in comments_df.columns and 'video_title' in comments_df.columns:
+            engagement_df = comments_df.groupby('video_title').size().to_frame('comment_count')
+            if videos_df is not None and not videos_df.empty:
+                merged = videos_df.set_index('video_title').join(engagement_df, lsuffix='_video', rsuffix='_comment')
+                # If 'comment_count' is missing, fill with 0
+                if 'comment_count' not in merged.columns:
+                    merged['comment_count'] = 0
+                # If 'view_count' is missing, fill with 1 to avoid division by zero
+                if 'view_count' not in merged.columns:
+                    merged['view_count'] = 1
+                merged['engagement_rate'] = merged['comment_count'] / merged['view_count']
+                merged = merged.sort_values('engagement_rate', ascending=False).head(10)
+                if not merged.empty:
+                    fig_engagement, ax = plt.subplots(figsize=(10, 6))
+                    merged['engagement_rate'].plot(kind='barh', ax=ax, color='mediumseagreen')
+                    ax.set_title("Top 10 Videos by Engagement Rate", fontproperties=BANGLA_FONT)
+                    ax.set_xlabel("Engagement Rate (Comments / Views)")
+                    ax.set_yticklabels(merged.index, fontproperties=BANGLA_FONT)
+                    plt.tight_layout()
+                    plt.close(fig_engagement)
+
+        # Comment activity over time
+        fig_time_series, ax = None, None
+        if 'published_date_comment' in comments_df.columns:
+            try:
+                comments_df['published_date_comment'] = pd.to_datetime(comments_df['published_date_comment'])
+                time_series = comments_df.set_index('published_date_comment').resample('D').size()
+                if not time_series.empty:
+                    fig_time_series, ax = plt.subplots(figsize=(10, 4))
+                    time_series.plot(ax=ax, color='darkorange')
+                    ax.set_title("Comment Activity Over Time", fontproperties=BANGLA_FONT)
+                    ax.set_xlabel("Date")
+                    ax.set_ylabel("Number of Comments")
+                    plt.tight_layout()
+                    plt.close(fig_time_series)
+            except Exception as e:
+                logger.error(f"Error in comment activity plot: {e}")
+
+        # Beautiful Bengali word cloud from YouTube comments
+        fig_wc, ax = None, None
+        if 'comment_text' in comments_df.columns:
+            text = " ".join(comment for comment in comments_df['comment_text'].astype(str))
+            text = clean_bengali_text(text)
+            for phrase, joined in PHRASES_TO_JOIN.items():
+                text = text.replace(phrase, joined)
+            try:
+                words = re.findall(r'[\u0980-\u09FF_]{2,}', text)
+                words = [w for w in words if w not in COMBINED_STOPWORDS]
+                words = [w for w in words if len(w) > 1]
+                words = [w for w in words if not re.search(r'[a-zA-Z]', w)]
+                from collections import Counter
+                word_freq = Counter(words)
+                min_freq = 2
+                most_common = set([w for w, _ in word_freq.most_common(3)])
+                filtered_words = [w for w in words if word_freq[w] >= min_freq and w not in most_common]
+                wc_text = " ".join(filtered_words)
+                wc = WordCloud(
+                    font_path=FONT_PATH,
+                    width=1600,
+                    height=900,
+                    background_color='white',
+                    stopwords=COMBINED_STOPWORDS,
+                    collocations=False,
+                    colormap='plasma',
+                    max_words=250,
+                    contour_width=2,
+                    contour_color='darkorange',
+                    regexp=r"[\u0980-\u09FF_]+"
+                ).generate(wc_text)
+                fig_wc, ax = plt.subplots(figsize=(15, 8))
+                ax.imshow(wc, interpolation='bilinear')
+                ax.axis("off")
+                ax.set_title("Bengali Word Cloud from YouTube Comments", fontproperties=BANGLA_FONT, fontsize=22)
+                plt.tight_layout()
+            except Exception as e:
+                logger.error(f"YouTube WordCloud failed: {e}")
 
-    return {**kpis, yt_channel_plot: fig_channels, yt_wordcloud_plot: fig_wc, yt_sentiment_pie_plot: fig_pie, yt_sentiment_by_video_plot: fig_sentiment_video}
+    return {
+        **kpis,
+        yt_channel_plot: fig_channels,
+        yt_wordcloud_plot: fig_wc,
+        'yt_top_videos_plot': fig_top_videos,
+        'yt_engagement_plot': fig_engagement,
+        'yt_comment_activity_plot': fig_comment_activity,
+        'yt_time_series_plot': fig_time_series
+    }
 
 def generate_youtube_topic_dashboard(videos_df_full_scan: pd.DataFrame):
     if videos_df_full_scan is None or videos_df_full_scan.empty: return None, None, None
     set_plot_style()
     
     channel_views = videos_df_full_scan.groupby('channel')['view_count'].sum().nlargest(15).sort_values()
-    fig_channel_views, ax = plt.subplots(figsize=(10, 7)); channel_views.plot(kind='barh', ax=ax, color='purple'); ax.set_title("Channel Dominance by Total Views (Top 15)", fontproperties=BANGLA_FONT); ax.set_xlabel("Combined Views on Topic"); ax.set_yticklabels(channel_views.index, fontproperties=BANGLA_FONT); plt.tight_layout()
+    fig_channel_views, ax = plt.subplots(figsize=(10, 7)); channel_views.plot(kind='barh', ax=ax, color='purple'); ax.set_title("Channel Dominance by Total Views (Top 15)", fontproperties=BANGLA_FONT); ax.set_xlabel("Combined Views on Topic"); ax.set_yticklabels(channel_views.index, fontproperties=BANGLA_FONT); plt.tight_layout(); plt.close(fig_channel_views)
 
     df_sample = videos_df_full_scan.sample(n=min(len(videos_df_full_scan), 200))
     avg_views, avg_engagement = df_sample['view_count'].median(), df_sample['engagement_rate'].median()
     fig_quadrant, ax = plt.subplots(figsize=(10, 8)); sns.scatterplot(data=df_sample, x='view_count', y='engagement_rate', size='like_count', sizes=(20, 400), hue='channel', alpha=0.7, ax=ax, legend=False)
     ax.set_xscale('log'); ax.set_yscale('log'); ax.set_title("Content Performance Quadrant", fontproperties=BANGLA_FONT); ax.set_xlabel("Video Views (Log Scale)", fontproperties=BANGLA_FONT); ax.set_ylabel("Engagement Rate (Log Scale)", fontproperties=BANGLA_FONT)
-    ax.axhline(avg_engagement, ls='--', color='gray'); ax.axvline(avg_views, ls='--', color='gray'); ax.text(avg_views*1.1, ax.get_ylim()[1], 'High Performers', color='green', fontproperties=BANGLA_FONT); ax.text(ax.get_xlim()[0], avg_engagement*1.1, 'Niche Stars', color='blue', fontproperties=BANGLA_FONT)
+    ax.axhline(avg_engagement, ls='--', color='gray'); ax.axvline(avg_views, ls='--', color='gray'); ax.text(avg_views*1.1, ax.get_ylim()[1], 'High Performers', color='green', fontproperties=BANGLA_FONT); ax.text(ax.get_xlim()[0], avg_engagement*1.1, 'Niche Stars', color='blue', fontproperties=BANGLA_FONT); plt.close(fig_quadrant)
 
     fig_age, ax = plt.subplots(figsize=(10, 7)); sns.scatterplot(data=df_sample, x='published_date', y='view_count', size='engagement_rate', sizes=(20, 400), alpha=0.6, ax=ax)
-    ax.set_yscale('log'); ax.set_title("Content Age vs. Impact", fontproperties=BANGLA_FONT); ax.set_xlabel("Publication Date", fontproperties=BANGLA_FONT); ax.set_ylabel("Views (Log Scale)", fontproperties=BANGLA_FONT); plt.xticks(rotation=45)
-    
+    ax.set_yscale('log'); ax.set_title("Content Age vs. Impact", fontproperties=BANGLA_FONT); ax.set_xlabel("Publication Date", fontproperties=BANGLA_FONT); ax.set_ylabel("Views (Log Scale)", fontproperties=BANGLA_FONT); plt.xticks(rotation=45); plt.close(fig_age)
     return fig_channel_views, fig_quadrant, fig_age
 
 # ==============================================================================
@@ -442,14 +569,14 @@ with gr.Blocks(theme=gr.themes.Soft(primary_hue="blue", secondary_hue="orange"),
             with gr.Row():
                 with gr.Column(scale=1):
                     gr.Markdown("### 1. Search Criteria")
-                    search_keywords_textbox = gr.Textbox(label="Search Keywords", placeholder="e.g., বিএনপি সমাবেশ")
+                    search_keywords_textbox = gr.Textbox(label="Search Keywords", placeholder="e.g.,ডাকসু ")
                     sites_to_search_textbox = gr.Textbox(label="Target Sites (Optional, comma-separated)", placeholder="e.g., prothomalo.com")
                     start_date_textbox = gr.Textbox(label="Start Date", placeholder="YYYY-MM-DD or 'last week'")
                     end_date_textbox = gr.Textbox(label="End Date", placeholder="YYYY-MM-DD or 'today'")
                     gr.Markdown("### 2. Scraping Parameters")
                     interval_days_slider = gr.Slider(1, 7, 3, step=1, label="Days per Interval")
                     max_pages_slider = gr.Slider(1, 10, 5, step=1, label="Max Pages per Interval")
-                    filter_keywords_textbox = gr.Textbox(label="Filter Keywords (comma-separated, optional)", placeholder="e.g., নির্বাচন, সরকার")
+                    filter_keywords_textbox = gr.Textbox(label="Filter Keywords (comma-separated, optional)", placeholder="e.g., নির্বাচন, ভিসি")
                     start_scraper_button = gr.Button("Start Scraping & Analysis", variant="primary")
                 with gr.Column(scale=2):
                     scraper_results_df = gr.DataFrame(label="Filtered Results", interactive=False, wrap=True)
@@ -475,33 +602,33 @@ with gr.Blocks(theme=gr.themes.Soft(primary_hue="blue", secondary_hue="orange"),
         with gr.TabItem("3. YouTube Topic Analysis", id=2):
             with gr.Row():
                 with gr.Column(scale=1):
-                    gr.Markdown("### 1. YouTube API & Search")
-                    yt_api_key = gr.Textbox(label="YouTube API Key", type="password", placeholder="Paste your API key")
-                    yt_search_keywords = gr.Textbox(label="Search Keywords", placeholder="e.g., বিএনপি, তারেক রহমান")
+                    gr.Markdown("### YouTube Search & Analysis")
+                    yt_api_key = gr.Textbox(label="YouTube API Key", placeholder="Paste your YouTube Data API v3 key here")
+                    yt_search_keywords = gr.Textbox(label="Search Keywords", placeholder="e.g.,বাংলাদেশ, নির্বাচন")
                     yt_published_after = gr.Textbox(label="Published After Date (Optional)", placeholder="YYYY-MM-DD or '1 month ago'")
-                    gr.Markdown("### 2. Analysis Parameters")
+                    gr.Markdown("### Analysis Parameters")
                     yt_max_videos_for_stats = gr.Slider(label="Videos to Scan for Topic Stats (Broad Scan)", minimum=50, maximum=750, value=300, step=50)
                     yt_num_videos_for_comments = gr.Slider(label="Top Videos for Comment Analysis (Deep Dive)", minimum=5, maximum=100, value=25, step=5)
                     yt_max_comments = gr.Slider(10, 100, 30, step=10, label="Max Comments per Video")
                     start_yt_analysis_button = gr.Button("Start YouTube Analysis", variant="primary")
                 with gr.Column(scale=2):
                     with gr.Group(visible=False) as yt_dashboard_group:
-                        gr.Markdown("### Topic Footprint KPIs (Based on Broad Scan)")
+                        gr.Markdown("### YouTube Topic Analytics Dashboard")
                         with gr.Row():
                             kpi_yt_total_topic_videos = gr.Textbox(label="Est. Total Videos on Topic (YT)", interactive=False)
                             kpi_yt_videos_found = gr.Textbox(label="Videos Scanned for Stats", interactive=False)
                             kpi_yt_views_scanned = gr.Textbox(label="Combined Views (of Scanned)", interactive=False)
                             kpi_yt_comments_scraped = gr.Textbox(label="Comments Analyzed (from Top Videos)", interactive=False)
                         with gr.Tabs():
-                            with gr.TabItem("Deep Dive Analysis (on Top Videos)"):
+                            with gr.TabItem("Top Videos & Engagement"):
                                 yt_videos_df_output = gr.DataFrame(label="Top Videos Analyzed for Comments (sorted by views)")
-                                with gr.Row():
-                                    yt_channel_plot = gr.Plot(label="Channel Contribution by Video Count")
-                                    yt_sentiment_pie_plot = gr.Plot(label="Overall Comment Sentiment")
-                                with gr.Row():
-                                    yt_wordcloud_plot = gr.Plot(label="Comment Word Cloud")
-                                    yt_sentiment_by_video_plot = gr.Plot(label="Comment Sentiment by Video")
-                            with gr.TabItem("Topic-Level Analytics (on All Scanned Videos)"):
+                                yt_top_videos_plot = gr.Plot(label="Top 10 Videos by Comment Count")
+                                yt_engagement_plot = gr.Plot(label="Top 10 Videos by Engagement Rate")
+                            with gr.TabItem("Comment Activity & Word Cloud"):
+                                yt_comment_activity_plot = gr.Plot(label="Comment Activity Over Time")
+                                yt_wordcloud_plot = gr.Plot(label="Bengali Word Cloud from Comments")
+                            with gr.TabItem("Channel & Topic Analytics"):
+                                yt_channel_plot = gr.Plot(label="Channel Contribution by Video Count")
                                 yt_channel_views_plot = gr.Plot(label="Channel Dominance by Views")
                                 yt_performance_quadrant_plot = gr.Plot(label="Content Performance Quadrant")
                                 yt_content_age_plot = gr.Plot(label="Content Age vs. Impact")
@@ -534,11 +661,22 @@ with gr.Blocks(theme=gr.themes.Soft(primary_hue="blue", secondary_hue="orange"),
 
     def update_news_dashboards(analyzed_df):
         if analyzed_df is None or analyzed_df.empty:
-            return {scraper_dashboard_group: gr.update(visible=False), sentiment_dashboard_tab: gr.update(visible=False)}
-        
+            return [gr.update(visible=False), '', '', '', None, None, None, gr.update(visible=False), None, None]
         scraper_updates = generate_scraper_dashboard(analyzed_df)
         sentiment_updates = generate_sentiment_dashboard(analyzed_df)
-        return {**scraper_updates, **sentiment_updates}
+        # Return outputs in the exact order of news_ui_components
+        return [
+            scraper_updates.get(scraper_dashboard_group, gr.update(visible=False)),
+            scraper_updates.get(kpi_total_articles, ''),
+            scraper_updates.get(kpi_unique_media, ''),
+            scraper_updates.get(kpi_date_range, ''),
+            scraper_updates.get(dashboard_timeline_plot, None),
+            scraper_updates.get(dashboard_media_plot, None),
+            scraper_updates.get(dashboard_wordcloud_plot, None),
+            sentiment_updates.get(sentiment_dashboard_tab, gr.update(visible=False)),
+            sentiment_updates.get(sentiment_pie_plot, None),
+            sentiment_updates.get(sentiment_by_media_plot, None)
+        ]
 
     news_ui_components = [
         scraper_dashboard_group, kpi_total_articles, kpi_unique_media, kpi_date_range,
@@ -572,26 +710,25 @@ with gr.Blocks(theme=gr.themes.Soft(primary_hue="blue", secondary_hue="orange"),
 
     def update_youtube_dashboards(results_data):
         if not results_data or results_data.get("full_scan") is None or results_data["full_scan"].empty:
-            return {
-                yt_dashboard_group: gr.update(visible=False), kpi_yt_total_topic_videos: "0",
-                kpi_yt_videos_found: "0", kpi_yt_views_scanned: "0", kpi_yt_comments_scraped: "0",
-                yt_channel_plot: None, yt_wordcloud_plot: None, yt_sentiment_pie_plot: None,
-                yt_sentiment_by_video_plot: None, yt_channel_views_plot: None,
-                yt_performance_quadrant_plot: None, yt_content_age_plot: None
-            }
-        
+            return [gr.update(visible=False), "0", "0", "0", "0", None, None, None, None, None, None, None]
         videos_df_full, comments_df, total_estimate = results_data.get("full_scan"), results_data.get("comments"), results_data.get("total_estimate", 0)
         deep_dive_updates = generate_youtube_dashboard(videos_df_full, comments_df)
         fig_ch_views, fig_quad, fig_age = generate_youtube_topic_dashboard(videos_df_full)
-        
-        return {
-            yt_dashboard_group: gr.update(visible=True),
-            kpi_yt_total_topic_videos: f"{total_estimate:,}",
-            **deep_dive_updates,
-            yt_channel_views_plot: fig_ch_views,
-            yt_performance_quadrant_plot: fig_quad,
-            yt_content_age_plot: fig_age,
-        }
+        # Return outputs in the exact order of yt_ui_components
+        return [
+            gr.update(visible=True),
+            f"{total_estimate:,}",
+            deep_dive_updates.get(kpi_yt_videos_found, "0"),
+            deep_dive_updates.get(kpi_yt_views_scanned, "0"),
+            deep_dive_updates.get(kpi_yt_comments_scraped, "0"),
+            deep_dive_updates.get(yt_channel_plot, None),
+            deep_dive_updates.get(yt_wordcloud_plot, None),
+            deep_dive_updates.get(yt_sentiment_pie_plot, None),
+            deep_dive_updates.get(yt_sentiment_by_video_plot, None),
+            fig_ch_views,
+            fig_quad,
+            fig_age
+        ]
     
     yt_ui_components = [
         yt_dashboard_group, kpi_yt_total_topic_videos, kpi_yt_videos_found, kpi_yt_views_scanned, kpi_yt_comments_scraped,
@@ -613,6 +750,6 @@ if __name__ == "__main__":
         logger.info("Using authentication credentials from environment variable.")
     else:
         logger.warning("No AUTH_CREDENTIALS found. Using default insecure credentials. Set this as an environment variable for production.")
-        auth_tuple = ("bnp", "12345")
+        auth_tuple = ("Arjon", "12345")
         
     app.launch(debug=True, auth=auth_tuple)