Update app.py

app.py

@@ -1,6 +1,7 @@
 import os
 import time
 import requests
+import json
 import pandas as pd
 import gradio as gr
 import plotly.express as px
@@ -13,11 +14,11 @@ API_TOKEN = os.getenv("LEADFEEDER_API_TOKEN")
 APP_PASSWORD = os.getenv("APP_PASSWORD")
 
 if not API_TOKEN or not APP_PASSWORD:
-    raise RuntimeError("Missing required secrets: LEADFEEDER_API_TOKEN or APP_PASSWORD")
+    raise RuntimeError("Missing required secrets")
 
-ACCOUNT_ID = "255333"  # PREMIUM account
+ACCOUNT_ID = "255333"
 BASE_URL = "https://api.leadfeeder.com"
-PAGE_SIZE = 50  # Reduced page size to prevent timeouts
+PAGE_SIZE = 50 
 
 HEADERS = {
     "Authorization": f"Token token={API_TOKEN}",
@@ -25,42 +26,30 @@ HEADERS = {
 }
 
 # =====================================================
-# UTILS: RATE LIMIT HANDLER
+# UTILS
 # =====================================================
 def make_request(url, params=None):
-    """
-    Wrapper to handle Leadfeeder's 100 req/min limit.
-    If we hit 429 (Too Many Requests), we sleep and retry.
-    """
     retries = 3
     while retries > 0:
-        r = requests.get(url, headers=HEADERS, params=params, timeout=30)
-        
+        r = requests.get(url, headers=HEADERS, params=params, timeout=45)
         if r.status_code == 429:
-            print("⏳ Rate limit hit. Sleeping for 60 seconds...")
             time.sleep(61)
             retries -= 1
             continue
-            
         r.raise_for_status()
         return r.json()
-    raise Exception("Max retries exceeded for rate limit")
+    raise Exception("Max retries exceeded")
 
 # =====================================================
 # DATA FETCHING
 # =====================================================
-def fetch_basic_leads(start_date, end_date, progress=gr.Progress()):
-    """
-    Step 1: Get the list of companies (Fast, but missing Landing Page)
-    """
+def fetch_basic_leads(start_date, end_date):
+    """ Step 1: Fast fetch of companies (no deep visit data yet) """
     page = 1
     rows = []
     
-    print(f"🚀 Fetching basic lead list for {start_date} to {end_date}...")
-    
     while True:
         try:
-            # We only ask for basic info here to be fast
             js = make_request(
                 f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads",
                 params={
@@ -71,12 +60,9 @@ def fetch_basic_leads(start_date, end_date, progress=gr.Progress()):
                     "include": "location" 
                 }
             )
-            
             data = js.get("data", [])
-            if not data:
-                break
+            if not data: break
                 
-            # Map Included Locations
             included = js.get("included", [])
             loc_map = {str(i["id"]): i["attributes"] for i in included if i["type"] == "locations"}
             
@@ -88,27 +74,21 @@ def fetch_basic_leads(start_date, end_date, progress=gr.Progress()):
                 rows.append({
                     "lead_id": lead.get("id"),
                     "company_name": a.get("name"),
-                    "website_url": a.get("website_url"),
                     "primary_industry": a.get("industry"),
                     "total_visits": a.get("visits"),
                     "lead_quality_score": a.get("quality"),
                     "last_visit_date": a.get("last_visit_date"),
                     "country": loc.get("country"),
                     "city": loc.get("city"),
-                    "region": loc.get("region"),
                     "revenue": a.get("revenue"),
                     "employee_count": a.get("employee_count"),
                     "assignee": a.get("assignee"),
                     "crm_organization_id": a.get("crm_organization_id"),
                     "linkedin_url": a.get("linkedin_url"),
-                    # Placeholders (Filled in Step 2)
-                    "landing_page_path": None,
-                    "exit_page_path": None
+                    "landing_page_path": None, # To be filled
+                    "exit_page_path": None     # To be filled
                 })
-            
-            print(f"✅ Page {page} loaded. Total leads: {len(rows)}")
             page += 1
-            
         except Exception as e:
             print(f"Error on page {page}: {e}")
             break
@@ -116,150 +96,130 @@ def fetch_basic_leads(start_date, end_date, progress=gr.Progress()):
     return pd.DataFrame(rows)
 
 def enrich_leads_with_visits(df, max_rows=None, progress=gr.Progress()):
-    """
-    Step 2: For each company, fetch the SPECIFIC visit details.
-    This is slow but GUARANTEES the landing page data.
-    """
-    if df.empty:
-        return df
+    """ Step 2: Deep fetch using JSON:API relationships """
+    if df.empty: return df
 
-    # If max_rows is set (e.g. for Preview), we only process top N rows
     target_df = df.head(max_rows) if max_rows else df
     total = len(target_df)
     
-    print(f"🕵️ Deep fetching visits for {total} companies...")
+    print(f"🕵️ Enriching {total} rows with deep visit data...")
     
-    # We iterate and update the dataframe
     for index, row in target_df.iterrows():
         lead_id = row["lead_id"]
-        
         try:
-            # Call the VISITS endpoint for this specific lead
-            # This endpoint returns the detailed visit history including page views
+            # CRITICAL CHANGE: We ask for page_views explicitly
             visit_data = make_request(
                 f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads/{lead_id}/visits",
-                params={"page[size]": 1} # We only need the latest 1 visit
+                params={"page[size]": 1, "include": "page_views"} 
             )
             
             visits = visit_data.get("data", [])
+            included = visit_data.get("included", [])
+            
             if visits:
-                # The visits endpoint usually returns the attributes directly on the visit object
-                # OR we look at the first page view if nested.
-                latest_visit = visits[0]["attributes"]
+                visit = visits[0] # Latest visit
+                v_attrs = visit.get("attributes", {})
                 
-                # Strategy: Check direct attributes first, then fallback
-                landing = (latest_visit.get("landing_page_path") or 
-                           latest_visit.get("entry_page") or 
-                           latest_visit.get("first_page_view", {}).get("url"))
-                           
-                exit_p = (latest_visit.get("exit_page_path") or 
-                          latest_visit.get("exit_page") or
-                          latest_visit.get("last_page_view", {}).get("url"))
+                # 1. Try Simple Attributes
+                landing = v_attrs.get("landing_page_path") or v_attrs.get("landing_page_url")
+                exit_p = v_attrs.get("exit_page_path") or v_attrs.get("exit_page_url")
+                
+                # 2. Try Deep Linking (Page Views)
+                if not landing or not exit_p:
+                    # Map all page views in the response
+                    pv_map = {p["id"]: p["attributes"] for p in included if p["type"] == "page_views"}
+                    
+                    # Get the list of page view IDs for this visit
+                    pv_ids = [r["id"] for r in visit.get("relationships", {}).get("page_views", {}).get("data", [])]
+                    
+                    if pv_ids:
+                        # Assuming order is chronological. 
+                        # First ID = Landing, Last ID = Exit
+                        first_pv = pv_map.get(pv_ids[0])
+                        last_pv = pv_map.get(pv_ids[-1])
+                        
+                        if not landing and first_pv:
+                            landing = first_pv.get("url") or first_pv.get("path")
+                        if not exit_p and last_pv:
+                            exit_p = last_pv.get("url") or last_pv.get("path")
 
-                # Update the main DataFrame
                 df.at[index, "landing_page_path"] = landing
                 df.at[index, "exit_page_path"] = exit_p
         
         except Exception as e:
-            print(f"Failed to enrich lead {lead_id}: {e}")
+            print(f"Failed lead {lead_id}: {e}")
             
-        # Update progress bar every 5 rows
         if index % 5 == 0:
-            progress(index / total, desc=f"Enriching row {index}/{total}")
+            progress(index / total, desc="Enriching...")
             
     return df
 
 # =====================================================
-# UI WRAPPERS
+# DEBUGGER
 # =====================================================
-def load_data_preview(start_date, end_date):
-    # 1. Get Basic List
-    df = fetch_basic_leads(start_date, end_date)
-    
-    if df.empty:
-        return df, pd.DataFrame(), "⚠️ No data found."
-    
-    # 2. Enrich ONLY the first 10 rows for the UI preview (Fast)
-    df_preview = df.copy()
-    df_preview = enrich_leads_with_visits(df_preview, max_rows=10)
-    
-    status = f"✅ Loaded **{len(df):,} companies**. Preview shows top 10 rows enriched with Landing Pages."
-    return df, df_preview.head(20), status
-
-def download_full_excel(df):
-    if df is None or df.empty:
-        return None
-    
-    # When downloading, we must enrich ALL rows (this takes time)
-    # Note: In a real web app, this should be a background job. 
-    # Here we do it inline with a progress bar.
-    print("⏳ Starting full enrichment for Excel export...")
-    enriched_df = enrich_leads_with_visits(df.copy()) # Process ALL rows
-    
-    path = "/tmp/leadfeeder_company_data_full.xlsx"
-    enriched_df.to_excel(path, index=False)
-    return path
+def inspect_raw_json(lead_id):
+    if not lead_id: return "Please enter a Lead ID"
+    try:
+        url = f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads/{lead_id}/visits"
+        params = {"page[size]": 1, "include": "page_views"}
+        r = requests.get(url, headers=HEADERS, params=params)
+        return json.dumps(r.json(), indent=2)
+    except Exception as e:
+        return str(e)
 
 # =====================================================
-# KPIS & DASHBOARDS (UNCHANGED)
+# UI LOGIC
 # =====================================================
-DASHBOARD_PRESETS = {
-    "Total Visits by Industry": ("primary_industry", "total_visits", "sum"),
-    "Total Visits by Country": ("country", "total_visits", "sum"),
-    "Avg Quality by Industry": ("primary_industry", "lead_quality_score", "mean"),
-    "Companies by Country": ("country", "company_name", "count"),
-}
-
-def build_kpis(df):
-    if df is None or df.empty: return 0, 0, 0, 0
-    return (len(df), int(df["total_visits"].sum()), round(df["lead_quality_score"].mean(), 2), round(df["crm_organization_id"].notna().mean() * 100, 1))
-
-def build_dashboard(df, preset, top_n):
-    if df is None or df.empty: return px.bar(title="No Data")
-    dim, metric, agg = DASHBOARD_PRESETS[preset]
-    if agg == "count": grouped = df.groupby(dim, dropna=False).size().reset_index(name="value")
-    else: grouped = df.groupby(dim, dropna=False)[metric].agg(agg).reset_index(name="value")
-    return px.bar(grouped.sort_values("value", ascending=False).head(top_n), x=dim, y="value", title=preset)
+def load_preview(start, end):
+    df = fetch_basic_leads(start, end)
+    if df.empty: return df, pd.DataFrame(), "No data."
+    
+    # Enrich top 10 for preview
+    df_prev = enrich_leads_with_visits(df.copy(), max_rows=10)
+    return df, df_prev.head(10), f"✅ Loaded {len(df)} leads. Previewing top 10."
+
+def download_excel(df):
+    if df is None: return None
+    # Enrich ALL for download
+    final_df = enrich_leads_with_visits(df.copy())
+    path = "/tmp/leadfeeder_full.xlsx"
+    final_df.to_excel(path, index=False)
+    return path
 
 # =====================================================
-# UI
+# UI LAYOUT
 # =====================================================
-with gr.Blocks(title="Leadfeeder Analytics") as demo:
-    gr.Markdown("## 🔐 Login")
-    pwd = gr.Textbox(type="password", label="App Password")
-    gr.Button("Login").click(lambda p: True if p == APP_PASSWORD else gr.Error("Invalid"), pwd, None)
-
+with gr.Blocks(title="Leadfeeder Pro Analytics") as demo:
+    gr.Markdown("## 🚀 Leadfeeder Analytics")
+    
+    with gr.Row():
+        pwd = gr.Textbox(type="password", label="Password")
+        gr.Button("Auth").click(lambda p: gr.Info("OK") if p==APP_PASSWORD else gr.Error("Bad Pwd"), pwd, None)
+    
     df_state = gr.State()
-    status = gr.Markdown()
-
+    
     with gr.Tabs():
-        with gr.Tab("📋 Data"):
-            with gr.Row():
-                start = gr.Textbox(label="Start", value=(date.today() - timedelta(days=30)).isoformat())
-                end = gr.Textbox(label="End", value=date.today().isoformat())
-            
-            load_btn = gr.Button("Load Data (Preview)", variant="primary")
+        with gr.Tab("📋 Data & Report"):
+            start = gr.Textbox(label="Start", value=(date.today()-timedelta(days=30)).isoformat())
+            end = gr.Textbox(label="End", value=date.today().isoformat())
             
             with gr.Row():
-                # NOTE: This button now triggers the FULL enrichment
-                dl_btn = gr.Button("⬇️ Enrich & Download Full Excel")
-                dl_file = gr.File()
+                btn_load = gr.Button("1. Load Data", variant="primary")
+                btn_dl = gr.Button("2. Enrich & Download Excel")
+                file_dl = gr.File()
             
-            preview = gr.Dataframe(label="Preview (Top 10 Enriched)")
+            msg = gr.Markdown()
+            table = gr.Dataframe(label="Preview (Top 10)")
             
-            load_btn.click(load_data_preview, [start, end], [df_state, preview, status])
-            dl_btn.click(download_full_excel, df_state, dl_file)
+            btn_load.click(load_preview, [start, end], [df_state, table, msg])
+            btn_dl.click(download_excel, df_state, file_dl)
 
-        with gr.Tab("📊 Dashboard"):
-            kpis = [gr.Number(label=l) for l in ["Companies", "Visits", "Quality", "CRM %"]]
-            gr.Button("Refresh KPIs").click(build_kpis, df_state, kpis)
-            
-            with gr.Row():
-                preset = gr.Dropdown(choices=list(DASHBOARD_PRESETS.keys()), label="Chart")
-                top_n = gr.Slider(5, 50, value=10, label="Top N")
-            
-            chart = gr.Plot()
-            gr.Button("Build").click(build_dashboard, [df_state, preset, top_n], chart)
+        with gr.Tab("🛠️ Debugger"):
+            gr.Markdown("Use this if columns are still NULL. Paste a 'lead_id' from the table above.")
+            dbg_id = gr.Textbox(label="Lead ID")
+            dbg_btn = gr.Button("Inspect Raw JSON")
+            dbg_out = gr.Code(language="json")
+            dbg_btn.click(inspect_raw_json, dbg_id, dbg_out)
 
 if __name__ == "__main__":
-    demo.launch(server_name="0.0.0.0", server_port=7860, ssr_mode=False)
+    demo.launch(server_name="0.0.0.0", server_port=7860)