Update app.py

app.py

@@ -3,8 +3,11 @@ import time
 import requests
 import json
 import pandas as pd
+import math
 import gradio as gr
 import plotly.express as px
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
 from datetime import date, timedelta
 
 # =====================================================
@@ -29,7 +32,7 @@ HEADERS = {
 CAMPAIGN_CONFIG_FILE = "campaign_rules.json"
 
 # =====================================================
-# 1. CAMPAIGN MANAGER & UTILS
+# 1. CAMPAIGN MANAGER & BUSINESS LOGIC
 # =====================================================
 DEFAULT_CAMPAIGNS = {
     "MS Stack": [
@@ -54,7 +57,6 @@ DEFAULT_CAMPAIGNS = {
 }
 
 def load_campaign_rules():
-    """Load campaign groups from JSON file or use defaults."""
     if os.path.exists(CAMPAIGN_CONFIG_FILE):
         try:
             with open(CAMPAIGN_CONFIG_FILE, "r") as f:
@@ -64,7 +66,6 @@ def load_campaign_rules():
     return DEFAULT_CAMPAIGNS
 
 def save_campaign_rules(rules_text):
-    """Parse JSON text from UI and save to file."""
     try:
         rules = json.loads(rules_text)
         with open(CAMPAIGN_CONFIG_FILE, "w") as f:
@@ -74,7 +75,6 @@ def save_campaign_rules(rules_text):
         return "❌ Error: Invalid JSON format.", None
 
 def categorize_quality(score):
-    """Classify Lead Quality Score into High/Mid/Low."""
     if pd.isna(score): return "Unknown"
     try:
         s = int(score)
@@ -86,10 +86,8 @@ def categorize_quality(score):
         return "Unknown"
 
 def assign_campaign(url, rules):
-    """Match URL against campaign keywords."""
     if not url or not isinstance(url, str):
         return "Uncategorized"
-    
     url_lower = url.lower()
     for campaign_name, keywords in rules.items():
         for kw in keywords:
@@ -98,29 +96,31 @@ def assign_campaign(url, rules):
     return "Other"
 
 def apply_business_logic(df):
-    """Apply Campaign and Quality logic to the DataFrame."""
     if df is None or df.empty: return df
     
-    # 1. Apply Quality Logic
-    df["Quality_Group"] = df["lead_quality_score"].apply(categorize_quality)
+    # 1. Quality Group
+    if "lead_quality_score" in df.columns:
+        df["Quality_Group"] = df["lead_quality_score"].apply(categorize_quality)
     
-    # 2. Apply Campaign Logic
+    # 2. Campaign Logic (requires landing_page_path)
     rules = load_campaign_rules()
-    # We use landing_page_path primarily.
-    df["Campaign"] = df["landing_page_path"].apply(lambda x: assign_campaign(x, rules))
-    
+    if "landing_page_path" in df.columns:
+        df["Campaign"] = df["landing_page_path"].apply(lambda x: assign_campaign(x, rules))
+    else:
+        df["Campaign"] = "Uncategorized"
+        
     return df
 
 # =====================================================
-# 2. PRESETS (Original + New)
+# 2. PRESETS
 # =====================================================
 DASHBOARD_PRESETS = {
-    # -- NEW PRESETS --
+    "All Campaigns (Multi-Chart)": ("SPECIAL_MULTI", None, None), # DYNAMIC ALL CAMPAIGNS
+    
     "Visits by Campaign": ("Campaign", "total_visits", "sum"),
     "Leads by Campaign": ("Campaign", "company_name", "count"),
     "Leads by Quality Group": ("Quality_Group", "company_name", "count"),
     
-    # -- ORIGINAL PRESETS --
     "Total Visits by Industry": ("primary_industry", "total_visits", "sum"),
     "Total Visits by Country": ("country", "total_visits", "sum"),
     "Avg Quality by Industry": ("primary_industry", "lead_quality_score", "mean"),
@@ -128,15 +128,13 @@ DASHBOARD_PRESETS = {
     "Companies by Country": ("country", "company_name", "count"),
     "Visits by City": ("city", "total_visits", "sum"),
     "Accounts by Assignee": ("assignee", "company_name", "count"),
-    "Visits by Assignee": ("assignee", "total_visits", "sum"),
 }
 
 TREND_PRESETS = {
-    # -- NEW TRENDS --
     "Visits Trend by Campaign": ("total_visits", "sum", "Campaign"),
+    "Leads Trend by Campaign": ("company_name", "count", "Campaign"),
     "Visits Trend by Quality": ("total_visits", "sum", "Quality_Group"),
     
-    # -- ORIGINAL TRENDS --
     "Total Visits Trend": ("total_visits", "sum", None),
     "Active Accounts Trend": ("company_name", "count", None),
     "Avg Lead Quality Trend": ("lead_quality_score", "mean", None),
@@ -146,7 +144,7 @@ TREND_PRESETS = {
 }
 
 # =====================================================
-# 3. API HANDLING
+# 3. API HANDLING (FULL DATA RESTORED)
 # =====================================================
 def make_request(url, params=None):
     retries = 3
@@ -162,7 +160,7 @@ def make_request(url, params=None):
     raise Exception("Max retries exceeded")
 
 def fetch_basic_leads(start_date, end_date):
-    """ Step 1: Fetch ALL company details (fast fetch). """
+    """ Fetch ALL company details (Restored all original fields). """
     page = 1
     rows = []
     print(f"🚀 Fetching full company list for {start_date} to {end_date}...")
@@ -202,17 +200,21 @@ def fetch_basic_leads(start_date, end_date):
                     "lead_quality_score": a.get("quality"),
                     "revenue": a.get("revenue"),
                     "employee_count": a.get("employee_count"),
+                    "employees_min": a.get("employees_range", {}).get("min") if a.get("employees_range") else None,
+                    "employees_max": a.get("employees_range", {}).get("max") if a.get("employees_range") else None,
                     "assignee": a.get("assignee"),
                     "emailed_to": a.get("emailed_to"),
+                    "crm_lead_id": a.get("crm_lead_id"),
                     "crm_organization_id": a.get("crm_organization_id"),
                     "tags": ", ".join(a.get("tags", [])) if a.get("tags") else None,
                     "linkedin_url": a.get("linkedin_url"),
+                    "twitter_handle": a.get("twitter_handle"),
+                    "facebook_url": a.get("facebook_url"),
                     "country": loc.get("country"),
                     "region": loc.get("region"),
                     "city": loc.get("city"),
                     "leadfeeder_url": a.get("view_in_leadfeeder"),
-                    # Placeholders for enrichment
-                    "landing_page_path": None,
+                    "landing_page_path": None, # Init for Enrichment
                     "exit_page_path": None
                 })
             print(f"✅ Page {page} loaded. Rows: {len(rows)}")
@@ -227,7 +229,7 @@ def fetch_basic_leads(start_date, end_date):
     return df
 
 def enrich_leads_with_visits(df, start_date, end_date, max_rows=None, progress=gr.Progress()):
-    """ Step 2: Deep enrich with visit_route parsing for Exit/Landing Page. """
+    """ Step 2: Deep enrich (Restored full logic). """
     if df.empty: return df
     target_df = df.head(max_rows) if max_rows else df
     total = len(target_df)
@@ -291,18 +293,15 @@ def enrich_leads_with_visits(df, start_date, end_date, max_rows=None, progress=g
     return df
 
 # =====================================================
-# 4. MAIN FUNCTIONS
+# 4. WRAPPERS
 # =====================================================
 def load_preview(start, end):
-    # 1. Fetch
     df = fetch_basic_leads(start, end)
     if df.empty: return df, pd.DataFrame(), "⚠️ No data found."
     
-    # 2. Enrich top 20 for preview speed
+    # Enrich top 20 for preview speed
     df_preview = df.copy()
     df_preview = enrich_leads_with_visits(df_preview, start, end, max_rows=20)
-    
-    # 3. Apply New Business Logic (Campaign/Quality)
     df_preview = apply_business_logic(df_preview)
     
     status = f"✅ Loaded **{len(df):,} companies**. Preview shows top 20 rows enriched."
@@ -312,10 +311,7 @@ def download_full_excel(df, start, end):
     if df is None or df.empty: return None
     print("⏳ Starting full enrichment for Excel export...")
     
-    # 1. Enrich ALL rows (Time consuming but necessary for Campaign logic)
     enriched_df = enrich_leads_with_visits(df.copy(), start, end)
-    
-    # 2. Apply Business Logic
     final_df = apply_business_logic(enriched_df)
     
     path = "/tmp/leadfeeder_campaign_data.xlsx"
@@ -336,8 +332,8 @@ def inspect_raw_json(lead_id, start_date, end_date):
 # 5. CHART ENGINES
 # =====================================================
 def build_kpis(df):
-    if df is None or df.empty: return 0, 0, 0, 0, 0, 0
-    # Apply logic if not present
+    if df is None or df.empty: return 0, 0, 0, 0, 0, 0, 0
+    
     if "Quality_Group" not in df.columns:
         df = apply_business_logic(df)
         
@@ -347,21 +343,60 @@ def build_kpis(df):
         int(df["total_visits"].sum()),
         round(df["lead_quality_score"].mean(), 2),
         round(df["crm_organization_id"].notna().mean() * 100, 1),
-        df[df["Quality_Group"] == "High Quality (8-10)"].shape[0], # Count of High Quality
+        round(df["linkedin_url"].notna().mean() * 100, 1), # Restored LinkedIn %
+        df[df["Quality_Group"] == "High Quality (8-10)"].shape[0],
     )
 
 def build_dashboard(df, preset, top_n):
     if df is None or df.empty: return px.bar(title="No Data")
     
-    # Ensure columns exist
-    if "Campaign" not in df.columns or "Quality_Group" not in df.columns:
-         # Warn if viewing campaign data without full enrichment
-         if "Campaign" in preset:
-             return px.bar(title="⚠️ Please Download Excel first to enrich Campaigns for all rows.")
-         df = apply_business_logic(df)
+    if "Campaign" not in df.columns: df = apply_business_logic(df)
 
+    # --- SPECIAL MULTI-CHART FOR ALL CAMPAIGNS ---
+    if preset == "All Campaigns (Multi-Chart)":
+        
+        # 1. Get ALL campaigns present in data
+        campaigns = df["Campaign"].dropna().unique().tolist()
+        if not campaigns: return px.bar(title="No Campaigns Found")
+        
+        # Sort them (maybe put 'Other' last)
+        campaigns.sort(key=lambda x: (x == "Other", x))
+        
+        num_cams = len(campaigns)
+        cols = 2 # 2 columns wide grid
+        rows = math.ceil(num_cams / cols)
+        
+        fig = make_subplots(
+            rows=rows, cols=cols, 
+            subplot_titles=campaigns,
+            vertical_spacing=0.1
+        )
+
+        for i, camp in enumerate(campaigns):
+            # Calculate grid position
+            r = (i // cols) + 1
+            c = (i % cols) + 1
+            
+            # Filter and Get Top Companies for this campaign
+            camp_df = df[df["Campaign"] == camp]
+            grouped = camp_df.groupby("company_name")["total_visits"].sum().reset_index()
+            grouped = grouped.sort_values("total_visits", ascending=False).head(top_n)
+            
+            # Add Bar Trace
+            fig.add_trace(
+                go.Bar(x=grouped["company_name"], y=grouped["total_visits"], name=camp),
+                row=r, col=c
+            )
+
+        fig.update_layout(
+            title_text=f"Top Companies by Campaign (All {num_cams})", 
+            height=300 * rows,  # Dynamic height
+            showlegend=False
+        )
+        return fig
+
+    # --- STANDARD CHARTS ---
     dim, metric, agg = DASHBOARD_PRESETS[preset]
-    
     if agg == "count":
         grouped = df.groupby(dim, dropna=False).size().reset_index(name="value")
     else:
@@ -375,41 +410,40 @@ def get_trend_filter_options(df, preset):
     metric, agg, segment = TREND_PRESETS[preset]
     if not segment: return gr.update(choices=[], value=None, visible=False)
     
-    # Ensure logic applied
     if segment not in df.columns: df = apply_business_logic(df)
         
     options = sorted(df[segment].astype(str).unique().tolist())
     return gr.update(choices=options, value=options[:5], visible=True, label=f"Filter {segment}")
 
 def build_trend(df, preset, grain, filter_values):
-    if df is None or df.empty: return px.line(title="No Data Loaded")
+    if df is None or df.empty: return px.line(title="No Data")
     
     metric, agg, segment = TREND_PRESETS[preset]
-    df = df.dropna(subset=["last_visit_date"]).copy()
+    df_t = df.dropna(subset=["last_visit_date"]).copy()
     
     if segment:
-        if segment not in df.columns: df = apply_business_logic(df)
+        if segment not in df_t.columns: df_t = apply_business_logic(df_t)
         if filter_values:
-            df = df[df[segment].isin(filter_values)]
+            df_t = df_t[df_t[segment].isin(filter_values)]
             
     if grain == "Weekly":
-        df["period"] = df["last_visit_date"].dt.to_period("W").astype(str)
+        df_t["period"] = df_t["last_visit_date"].dt.to_period("W").astype(str)
     elif grain == "Monthly":
-        df["period"] = df["last_visit_date"].dt.to_period("M").astype(str)
+        df_t["period"] = df_t["last_visit_date"].dt.to_period("M").astype(str)
     else:
-        df["period"] = df["last_visit_date"].dt.date
+        df_t["period"] = df_t["last_visit_date"].dt.date
         
     val = metric
     if agg == "count":
-        df["_v"] = 1
+        df_t["_v"] = 1
         val = "_v"
         
     if segment:
-        ts = df.groupby(["period", segment])[val].agg(agg).reset_index()
-        return px.line(ts, x="period", y=val, color=segment, template="plotly_white", title=preset)
+        ts = df_t.groupby(["period", segment])[val].agg(agg).reset_index()
+        return px.line(ts, x="period", y=val, color=segment, title=preset)
     
-    ts = df.groupby("period")[val].agg(agg).reset_index()
-    return px.line(ts, x="period", y=val, markers=True, template="plotly_white", title=preset)
+    ts = df_t.groupby("period")[val].agg(agg).reset_index()
+    return px.line(ts, x="period", y=val, markers=True, title=preset)
 
 # =====================================================
 # 6. UI LAYOUT
@@ -442,12 +476,12 @@ with gr.Blocks(title="Leadfeeder Campaign Pro") as demo:
 
         # --- TAB 2: DASHBOARD ---
         with gr.Tab("📊 Dashboard"):
-            kpis = [gr.Number(label=l) for l in ["Total Companies", "Active Companies", "Total Visits", "Avg Quality", "CRM %", "High Quality Leads (8-10)"]]
+            kpis = [gr.Number(label=l) for l in ["Total Companies", "Active Companies", "Total Visits", "Avg Quality", "CRM %", "LinkedIn %", "High Quality (8-10)"]]
             gr.Button("Refresh KPIs").click(build_kpis, df_state, kpis)
             
             with gr.Row():
-                preset = gr.Dropdown(choices=list(DASHBOARD_PRESETS.keys()), label="Chart", value="Visits by Campaign")
-                top_n = gr.Slider(5, 50, value=10, label="Top N")
+                preset = gr.Dropdown(choices=list(DASHBOARD_PRESETS.keys()), label="Chart", value="All Campaigns (Multi-Chart)")
+                top_n = gr.Slider(5, 50, value=5, label="Top N per Chart")
             
             chart = gr.Plot()
             gr.Button("Build View").click(build_dashboard, [df_state, preset, top_n], chart)
@@ -464,17 +498,13 @@ with gr.Blocks(title="Leadfeeder Campaign Pro") as demo:
             trend_view.change(get_trend_filter_options, [df_state, trend_view], filter_dropdown)
             gr.Button("Show Trend", variant="primary").click(build_trend, [df_state, trend_view, grain, filter_dropdown], plot)
 
-        # --- TAB 4: SETTINGS (NEW) ---
+        # --- TAB 4: SETTINGS ---
         with gr.Tab("⚙️ Campaign Settings"):
             gr.Markdown("### Manage Campaign Groups & Keywords")
-            gr.Markdown("Modify the JSON below to add new groups or keywords. Click 'Save' to apply changes.")
-            
-            # Load current rules into text box
             current_rules = json.dumps(load_campaign_rules(), indent=4)
             config_editor = gr.Code(value=current_rules, language="json", label="Campaign Configuration")
             save_config_btn = gr.Button("💾 Save Configuration")
             config_status = gr.Markdown()
-            
             save_config_btn.click(save_campaign_rules, config_editor, [config_status, gr.State()])
 
         # --- TAB 5: DEBUGGER ---