Update app.py

app.py

@@ -64,12 +64,17 @@ def authenticate(pwd):
     return True
 
 # =====================================================
-# DATA FETCH (FIXED FOR VISITS)
+# DATA FETCH (DEEP DIVE FOR PAGE VIEWS)
 # =====================================================
-def fetch_company_data(start_date, end_date):
+def fetch_company_data(start_date, end_date, progress=gr.Progress()):
     page = 1
     rows = []
     
+    # We ask for "visits.page_views" to get the nested URL data
+    include_param = "location,visits,visits.page_views"
+    
+    print(f"🚀 Starting fetch for {start_date} to {end_date}...")
+
     while True:
         try:
             r = requests.get(
@@ -80,9 +85,9 @@ def fetch_company_data(start_date, end_date):
                     "end_date": end_date,
                     "page[number]": page,
                     "page[size]": PAGE_SIZE,
-                    "include": "location,visits"  # <--- CRITICAL: Request visits data
+                    "include": include_param 
                 },
-                timeout=30
+                timeout=45 # Increased timeout for heavier query
             )
             r.raise_for_status()
             js = r.json()
@@ -94,41 +99,67 @@ def fetch_company_data(start_date, end_date):
             # --- MAP INCLUDED DATA ---
             included = js.get("included", [])
             
-            # 1. Map Locations by ID
+            # 1. Map Locations
             loc_map = {
-                i["id"]: i["attributes"]
+                str(i["id"]): i["attributes"]
                 for i in included
                 if i["type"] == "locations"
             }
             
-            # 2. Map Visits by ID (to lookup landing pages)
+            # 2. Map Visits
             visit_map = {
-                i["id"]: i["attributes"]
+                str(i["id"]): i["attributes"]
                 for i in included
                 if i["type"] == "visits"
             }
             
+            # 3. Map Page Views (This is where the URL lives)
+            page_view_map = {
+                str(i["id"]): i["attributes"]
+                for i in included
+                if i["type"] == "page_views"
+            }
+            
             for lead in data:
                 a = lead["attributes"]
                 
                 # --- LOGIC: GET LATEST VISIT ---
-                landing = None
-                exit_page = None
+                landing_url = None
+                exit_url = None
                 
-                # The 'relationships' key contains a list of visit IDs for this company
+                # Get visit IDs 
                 visit_refs = lead.get("relationships", {}).get("visits", {}).get("data", [])
                 
                 if visit_refs:
-                    # We take the last visit in the list, assuming it is the most recent
-                    latest_visit_id = visit_refs[-1]["id"]
+                    # Get the ID of the LAST visit (Most recent)
+                    latest_visit_id = str(visit_refs[-1]["id"])
                     
-                    # Look up the details in our map
-                    visit_details = visit_map.get(latest_visit_id, {})
-                    landing = visit_details.get("landing_page_path")
-                    exit_page = visit_details.get("exit_page_path")
-                
+                    # Find that visit in the included list to check its relationships
+                    # Note: We need the RELATIONSHIP object for the visit, not just attributes
+                    # We have to scan 'included' again or build a relationship map. 
+                    # Optimization: We loop through included only if needed or build a full map.
+                    
+                    # Efficient approach: Find the visit object in 'included' list directly
+                    visit_obj = next((item for item in included if str(item["id"]) == latest_visit_id and item["type"] == "visits"), None)
+                    
+                    if visit_obj:
+                        # Get Page Views for this specific visit
+                        pv_refs = visit_obj.get("relationships", {}).get("page_views", {}).get("data", [])
+                        
+                        if pv_refs:
+                            # FIRST Page View = Landing Page
+                            first_pv_id = str(pv_refs[0]["id"])
+                            if first_pv_id in page_view_map:
+                                landing_url = page_view_map[first_pv_id].get("url") or page_view_map[first_pv_id].get("path")
+
+                            # LAST Page View = Exit Page
+                            last_pv_id = str(pv_refs[-1]["id"])
+                            if last_pv_id in page_view_map:
+                                exit_url = page_view_map[last_pv_id].get("url") or page_view_map[last_pv_id].get("path")
+
                 # --- LOGIC: GET LOCATION ---
-                loc_id = lead.get("relationships", {}).get("location", {}).get("data", {}).get("id")
+                loc_id_ref = lead.get("relationships", {}).get("location", {}).get("data", {})
+                loc_id = str(loc_id_ref.get("id")) if loc_id_ref else None
                 loc = loc_map.get(loc_id, {})
                 
                 rows.append({
@@ -146,10 +177,8 @@ def fetch_company_data(start_date, end_date):
                     "total_visits": a.get("visits"),
                     "lead_quality_score": a.get("quality"),
                     "employee_count": a.get("employee_count"),
-                    "employees_min": a.get("employees_range", {}).get("min") 
-                        if a.get("employees_range") else None,
-                    "employees_max": a.get("employees_range", {}).get("max") 
-                        if a.get("employees_range") else None,
+                    "employees_min": a.get("employees_range", {}).get("min") if a.get("employees_range") else None,
+                    "employees_max": a.get("employees_range", {}).get("max") if a.get("employees_range") else None,
                     "revenue": a.get("revenue"),
                     "assignee": a.get("assignee"),
                     "emailed_to": a.get("emailed_to"),
@@ -161,8 +190,8 @@ def fetch_company_data(start_date, end_date):
                     "facebook_url": a.get("facebook_url"),
                     
                     # --- NEW FIELDS ---
-                    "landing_page_path": landing,
-                    "exit_page_path": exit_page,
+                    "landing_page_path": landing_url,
+                    "exit_page_path": exit_url,
                     # ------------------
                     
                     "country": loc.get("country"),
@@ -171,10 +200,14 @@ def fetch_company_data(start_date, end_date):
                     "leadfeeder_url": a.get("view_in_leadfeeder"),
                 })
             
+            print(f"✅ Page {page} processed. Total rows: {len(rows)}")
             page += 1
             
         except requests.exceptions.RequestException as e:
             print(f"Error fetching page {page}: {e}")
+            # If the complex include fails (400 Bad Request), you might see it here.
+            if "400" in str(e):
+                print("❌ The API rejected the nested include. Your plan might not support 'visits.page_views'.")
             break
 
     df = pd.DataFrame(rows)
@@ -183,7 +216,7 @@ def fetch_company_data(start_date, end_date):
     return df
 
 # =====================================================
-# LOAD WRAPPER (STATE SAFE)
+# LOAD WRAPPER
 # =====================================================
 def load_data_with_preview(start_date, end_date):
     df = fetch_company_data(start_date, end_date)