Update app.py

app.py

@@ -14,9 +14,9 @@ API_TOKEN = os.getenv("LEADFEEDER_API_TOKEN")
 APP_PASSWORD = os.getenv("APP_PASSWORD")
 
 if not API_TOKEN or not APP_PASSWORD:
-    raise RuntimeError("Missing required secrets")
+    raise RuntimeError("Missing required secrets: LEADFEEDER_API_TOKEN or APP_PASSWORD")
 
-ACCOUNT_ID = "255333"
+ACCOUNT_ID = "255333"  # PREMIUM account
 BASE_URL = "https://api.leadfeeder.com"
 PAGE_SIZE = 50 
 
@@ -26,13 +26,14 @@ HEADERS = {
 }
 
 # =====================================================
-# UTILS
+# UTILS: RATE LIMIT HANDLER
 # =====================================================
 def make_request(url, params=None):
     retries = 3
     while retries > 0:
         r = requests.get(url, headers=HEADERS, params=params, timeout=45)
         if r.status_code == 429:
+            print("⏳ Rate limit hit. Sleeping for 60 seconds...")
             time.sleep(61)
             retries -= 1
             continue
@@ -44,10 +45,14 @@ def make_request(url, params=None):
 # DATA FETCHING
 # =====================================================
 def fetch_basic_leads(start_date, end_date):
-    """ Step 1: Fast fetch of companies (no deep visit data yet) """
+    """ 
+    Step 1: Fetch ALL company details (fast fetch). 
+    """
     page = 1
     rows = []
     
+    print(f"🚀 Fetching full company list for {start_date} to {end_date}...")
+    
     while True:
         try:
             js = make_request(
@@ -61,8 +66,10 @@ def fetch_basic_leads(start_date, end_date):
                 }
             )
             data = js.get("data", [])
-            if not data: break
+            if not data:
+                break
                 
+            # Map Included Locations
             included = js.get("included", [])
             loc_map = {str(i["id"]): i["attributes"] for i in included if i["type"] == "locations"}
             
@@ -74,80 +81,106 @@ def fetch_basic_leads(start_date, end_date):
                 rows.append({
                     "lead_id": lead.get("id"),
                     "company_name": a.get("name"),
+                    "website_url": a.get("website_url"),
+                    "phone": a.get("phone"),
+                    "business_id": a.get("business_id"),
                     "primary_industry": a.get("industry"),
+                    "all_industries": ", ".join([i.get("name") for i in a.get("industries", [])]) if a.get("industries") else None,
+                    "first_visit_date": a.get("first_visit_date"),
+                    "last_visit_date": a.get("last_visit_date"),
                     "total_visits": a.get("visits"),
                     "lead_quality_score": a.get("quality"),
-                    "last_visit_date": a.get("last_visit_date"),
-                    "country": loc.get("country"),
-                    "city": loc.get("city"),
                     "revenue": a.get("revenue"),
                     "employee_count": a.get("employee_count"),
+                    "employees_min": a.get("employees_range", {}).get("min") if a.get("employees_range") else None,
+                    "employees_max": a.get("employees_range", {}).get("max") if a.get("employees_range") else None,
                     "assignee": a.get("assignee"),
+                    "emailed_to": a.get("emailed_to"),
+                    "crm_lead_id": a.get("crm_lead_id"),
                     "crm_organization_id": a.get("crm_organization_id"),
+                    "tags": ", ".join(a.get("tags", [])) if a.get("tags") else None,
                     "linkedin_url": a.get("linkedin_url"),
-                    "landing_page_path": None, # To be filled
-                    "exit_page_path": None     # To be filled
+                    "twitter_handle": a.get("twitter_handle"),
+                    "facebook_url": a.get("facebook_url"),
+                    "country": loc.get("country"),
+                    "region": loc.get("region"),
+                    "city": loc.get("city"),
+                    "leadfeeder_url": a.get("view_in_leadfeeder"),
+                    "landing_page_path": None,
+                    "exit_page_path": None
                 })
+            
+            print(f"✅ Page {page} loaded. Rows: {len(rows)}")
             page += 1
+            
         except Exception as e:
             print(f"Error on page {page}: {e}")
             break
 
-    return pd.DataFrame(rows)
+    df = pd.DataFrame(rows)
+    if not df.empty:
+        df["last_visit_date"] = pd.to_datetime(df["last_visit_date"], errors="coerce")
+    return df
 
-def enrich_leads_with_visits(df, max_rows=None, progress=gr.Progress()):
-    """ Step 2: Deep fetch using JSON:API relationships """
-    if df.empty: return df
+def enrich_leads_with_visits(df, start_date, end_date, max_rows=None, progress=gr.Progress()):
+    """ 
+    Step 2: Go back and fetch the specific Visit URLs for each company.
+    CRITICAL FIX: Now accepts start_date and end_date.
+    """
+    if df.empty:
+        return df
 
     target_df = df.head(max_rows) if max_rows else df
     total = len(target_df)
     
-    print(f"🕵️ Enriching {total} rows with deep visit data...")
+    print(f"🕵️ Deep enriching {total} rows with visit data ({start_date} to {end_date})...")
     
     for index, row in target_df.iterrows():
         lead_id = row["lead_id"]
+        
         try:
-            # CRITICAL CHANGE: We ask for page_views explicitly
+            # FIX: Added start_date and end_date to params!
             visit_data = make_request(
                 f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads/{lead_id}/visits",
-                params={"page[size]": 1, "include": "page_views"} 
+                params={
+                    "start_date": start_date,
+                    "end_date": end_date,
+                    "page[size]": 1, 
+                    "include": "page_views"
+                } 
             )
             
             visits = visit_data.get("data", [])
             included = visit_data.get("included", [])
             
+            landing = None
+            exit_p = None
+            
             if visits:
-                visit = visits[0] # Latest visit
+                visit = visits[0]
                 v_attrs = visit.get("attributes", {})
                 
-                # 1. Try Simple Attributes
+                # A. Direct Attributes
                 landing = v_attrs.get("landing_page_path") or v_attrs.get("landing_page_url")
                 exit_p = v_attrs.get("exit_page_path") or v_attrs.get("exit_page_url")
                 
-                # 2. Try Deep Linking (Page Views)
+                # B. Page Views (Deep Link)
                 if not landing or not exit_p:
-                    # Map all page views in the response
                     pv_map = {p["id"]: p["attributes"] for p in included if p["type"] == "page_views"}
-                    
-                    # Get the list of page view IDs for this visit
                     pv_ids = [r["id"] for r in visit.get("relationships", {}).get("page_views", {}).get("data", [])]
                     
                     if pv_ids:
-                        # Assuming order is chronological. 
-                        # First ID = Landing, Last ID = Exit
                         first_pv = pv_map.get(pv_ids[0])
+                        if first_pv: landing = first_pv.get("url") or first_pv.get("path")
+                            
                         last_pv = pv_map.get(pv_ids[-1])
-                        
-                        if not landing and first_pv:
-                            landing = first_pv.get("url") or first_pv.get("path")
-                        if not exit_p and last_pv:
-                            exit_p = last_pv.get("url") or last_pv.get("path")
+                        if last_pv: exit_p = last_pv.get("url") or last_pv.get("path")
 
                 df.at[index, "landing_page_path"] = landing
                 df.at[index, "exit_page_path"] = exit_p
         
         except Exception as e:
-            print(f"Failed lead {lead_id}: {e}")
+            print(f"Failed to enrich lead {lead_id}: {e}")
             
         if index % 5 == 0:
             progress(index / total, desc="Enriching...")
@@ -155,13 +188,19 @@ def enrich_leads_with_visits(df, max_rows=None, progress=gr.Progress()):
     return df
 
 # =====================================================
-# DEBUGGER
+# DEBUGGER TOOL
 # =====================================================
-def inspect_raw_json(lead_id):
+def inspect_raw_json(lead_id, start_date, end_date):
     if not lead_id: return "Please enter a Lead ID"
     try:
+        # FIX: Added dates to Debugger URL params
         url = f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads/{lead_id}/visits"
-        params = {"page[size]": 1, "include": "page_views"}
+        params = {
+            "start_date": start_date, 
+            "end_date": end_date, 
+            "page[size]": 1, 
+            "include": "page_views"
+        }
         r = requests.get(url, headers=HEADERS, params=params)
         return json.dumps(r.json(), indent=2)
     except Exception as e:
@@ -172,20 +211,59 @@ def inspect_raw_json(lead_id):
 # =====================================================
 def load_preview(start, end):
     df = fetch_basic_leads(start, end)
-    if df.empty: return df, pd.DataFrame(), "No data."
+    if df.empty:
+        return df, pd.DataFrame(), "⚠️ No data found."
     
-    # Enrich top 10 for preview
-    df_prev = enrich_leads_with_visits(df.copy(), max_rows=10)
-    return df, df_prev.head(10), f"✅ Loaded {len(df)} leads. Previewing top 10."
-
-def download_excel(df):
-    if df is None: return None
-    # Enrich ALL for download
-    final_df = enrich_leads_with_visits(df.copy())
-    path = "/tmp/leadfeeder_full.xlsx"
-    final_df.to_excel(path, index=False)
+    # Pass dates to enrichment!
+    df_preview = df.copy()
+    df_preview = enrich_leads_with_visits(df_preview, start, end, max_rows=10)
+    
+    status = f"✅ Loaded **{len(df):,} companies**. Preview shows top 10 rows enriched."
+    return df, df_preview.head(10), status
+
+def download_full_excel(df, start, end):
+    if df is None or df.empty: return None
+    
+    print("⏳ Starting full enrichment for Excel export...")
+    # Pass dates to enrichment!
+    enriched_df = enrich_leads_with_visits(df.copy(), start, end) 
+    
+    path = "/tmp/leadfeeder_company_data_full.xlsx"
+    enriched_df.to_excel(path, index=False)
     return path
 
+# =====================================================
+# DASHBOARD LOGIC (KPIs)
+# =====================================================
+DASHBOARD_PRESETS = {
+    "Total Visits by Industry": ("primary_industry", "total_visits", "sum"),
+    "Total Visits by Country": ("country", "total_visits", "sum"),
+    "Avg Quality by Industry": ("primary_industry", "lead_quality_score", "mean"),
+    "Top Accounts by Visits": ("company_name", "total_visits", "sum"),
+    "Companies by Country": ("country", "company_name", "count"),
+    "Visits by City": ("city", "total_visits", "sum"),
+}
+
+def build_kpis(df):
+    if df is None or df.empty: return 0, 0, 0, 0, 0, 0
+    return (
+        len(df),
+        df["total_visits"].gt(0).sum(),
+        int(df["total_visits"].sum()),
+        round(df["lead_quality_score"].mean(), 2),
+        round(df["crm_organization_id"].notna().mean() * 100, 1),
+        round(df["linkedin_url"].notna().mean() * 100, 1),
+    )
+
+def build_dashboard(df, preset, top_n):
+    if df is None or df.empty: return px.bar(title="No Data")
+    dim, metric, agg = DASHBOARD_PRESETS[preset]
+    if agg == "count":
+        grouped = df.groupby(dim, dropna=False).size().reset_index(name="value")
+    else:
+        grouped = df.groupby(dim, dropna=False)[metric].agg(agg).reset_index(name="value")
+    return px.bar(grouped.sort_values("value", ascending=False).head(top_n), x=dim, y="value", title=preset)
+
 # =====================================================
 # UI LAYOUT
 # =====================================================
@@ -193,33 +271,50 @@ with gr.Blocks(title="Leadfeeder Pro Analytics") as demo:
     gr.Markdown("## 🚀 Leadfeeder Analytics")
     
     with gr.Row():
-        pwd = gr.Textbox(type="password", label="Password")
-        gr.Button("Auth").click(lambda p: gr.Info("OK") if p==APP_PASSWORD else gr.Error("Bad Pwd"), pwd, None)
+        pwd = gr.Textbox(type="password", label="App Password")
+        gr.Button("Auth").click(lambda p: gr.Info("Success") if p==APP_PASSWORD else gr.Error("Invalid"), pwd, None)
     
     df_state = gr.State()
+    status = gr.Markdown()
     
     with gr.Tabs():
+        # --- TAB 1: DATA ---
         with gr.Tab("📋 Data & Report"):
-            start = gr.Textbox(label="Start", value=(date.today()-timedelta(days=30)).isoformat())
-            end = gr.Textbox(label="End", value=date.today().isoformat())
+            with gr.Row():
+                start = gr.Textbox(label="Start Date", value=(date.today()-timedelta(days=30)).isoformat())
+                end = gr.Textbox(label="End Date", value=date.today().isoformat())
             
             with gr.Row():
-                btn_load = gr.Button("1. Load Data", variant="primary")
-                btn_dl = gr.Button("2. Enrich & Download Excel")
-                file_dl = gr.File()
+                btn_load = gr.Button("1. Load Data (Preview)", variant="primary")
+                btn_dl = gr.Button("2. Enrich & Download Full Excel")
+                file_dl = gr.File(label="Download Excel")
+            
+            table = gr.Dataframe(label="Preview (Top 10 Enriched)", interactive=True)
             
-            msg = gr.Markdown()
-            table = gr.Dataframe(label="Preview (Top 10)")
+            btn_load.click(load_preview, [start, end], [df_state, table, status])
+            # Updated to pass start/end inputs
+            btn_dl.click(download_full_excel, [df_state, start, end], file_dl)
+
+        # --- TAB 2: DASHBOARD ---
+        with gr.Tab("📊 Dashboard"):
+            kpis = [gr.Number(label=l) for l in ["Total Companies", "Active Companies", "Total Visits", "Avg Quality", "CRM %", "LinkedIn %"]]
+            gr.Button("Refresh KPIs").click(build_kpis, df_state, kpis)
+            
+            with gr.Row():
+                preset = gr.Dropdown(choices=list(DASHBOARD_PRESETS.keys()), label="Chart")
+                top_n = gr.Slider(5, 50, value=10, label="Top N")
             
-            btn_load.click(load_preview, [start, end], [df_state, table, msg])
-            btn_dl.click(download_excel, df_state, file_dl)
+            chart = gr.Plot()
+            gr.Button("Build View").click(build_dashboard, [df_state, preset, top_n], chart)
 
+        # --- TAB 3: DEBUGGER ---
         with gr.Tab("🛠️ Debugger"):
-            gr.Markdown("Use this if columns are still NULL. Paste a 'lead_id' from the table above.")
-            dbg_id = gr.Textbox(label="Lead ID")
+            gr.Markdown("Use this to inspect raw API data for a specific lead if columns are empty.")
+            dbg_id = gr.Textbox(label="Lead ID (Copy from Data tab)")
             dbg_btn = gr.Button("Inspect Raw JSON")
             dbg_out = gr.Code(language="json")
-            dbg_btn.click(inspect_raw_json, dbg_id, dbg_out)
+            # Updated to pass start/end inputs
+            dbg_btn.click(inspect_raw_json, [dbg_id, start, end], dbg_out)
 
 if __name__ == "__main__":
     demo.launch(server_name="0.0.0.0", server_port=7860)