mktgtech commited on
Commit
1d750ec
·
verified ·
1 Parent(s): 837bfde

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +327 -231
app.py CHANGED
@@ -3,9 +3,10 @@ import time
3
  import requests
4
  import json
5
  import pandas as pd
6
- import gradio as gr
7
  import plotly.express as px
8
  from datetime import date, timedelta
 
9
 
10
  # =====================================================
11
  # CONFIG / SECRETS
@@ -14,11 +15,14 @@ API_TOKEN = os.getenv("LEADFEEDER_API_TOKEN")
14
  APP_PASSWORD = os.getenv("APP_PASSWORD")
15
 
16
  if not API_TOKEN or not APP_PASSWORD:
17
- raise RuntimeError("Missing required secrets: LEADFEEDER_API_TOKEN or APP_PASSWORD")
 
 
 
18
 
19
- ACCOUNT_ID = "255333" # PREMIUM account
20
  BASE_URL = "https://api.leadfeeder.com"
21
- PAGE_SIZE = 100
22
 
23
  HEADERS = {
24
  "Authorization": f"Token token={API_TOKEN}",
@@ -26,7 +30,7 @@ HEADERS = {
26
  }
27
 
28
  # =====================================================
29
- # PRESETS
30
  # =====================================================
31
  DASHBOARD_PRESETS = {
32
  "Total Visits by Industry": ("primary_industry", "total_visits", "sum"),
@@ -48,15 +52,11 @@ TREND_PRESETS = {
48
  "Visits Trend by Assignee": ("total_visits", "sum", "assignee"),
49
  }
50
 
51
- # =====================================================
52
- # UTILS: RATE LIMIT HANDLER
53
- # =====================================================
54
  def make_request(url, params=None):
55
  retries = 3
56
  while retries > 0:
57
  r = requests.get(url, headers=HEADERS, params=params, timeout=45)
58
  if r.status_code == 429:
59
- print("⏳ Rate limit hit. Sleeping for 60 seconds...")
60
  time.sleep(61)
61
  retries -= 1
62
  continue
@@ -64,32 +64,18 @@ def make_request(url, params=None):
64
  return r.json()
65
  raise Exception("Max retries exceeded")
66
 
67
- # =====================================================
68
- # DATA FETCHING
69
- # =====================================================
70
  def fetch_basic_leads(start_date, end_date):
71
- """ Step 1: Fetch ALL company details (fast fetch). """
72
  page = 1
73
  rows = []
74
-
75
- print(f"🚀 Fetching full company list for {start_date} to {end_date}...")
76
-
77
  while True:
78
  try:
79
  js = make_request(
80
  f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads",
81
- params={
82
- "start_date": start_date,
83
- "end_date": end_date,
84
- "page[number]": page,
85
- "page[size]": PAGE_SIZE,
86
- "include": "location"
87
- }
88
  )
89
  data = js.get("data", [])
90
- if not data:
91
- break
92
-
93
  included = js.get("included", [])
94
  loc_map = {str(i["id"]): i["attributes"] for i in included if i["type"] == "locations"}
95
 
@@ -97,290 +83,400 @@ def fetch_basic_leads(start_date, end_date):
97
  a = lead["attributes"]
98
  loc_id = lead.get("relationships", {}).get("location", {}).get("data", {}).get("id")
99
  loc = loc_map.get(str(loc_id), {})
100
-
101
  rows.append({
102
  "lead_id": lead.get("id"),
103
  "company_name": a.get("name"),
104
  "website_url": a.get("website_url"),
105
- "phone": a.get("phone"),
106
- "business_id": a.get("business_id"),
107
  "primary_industry": a.get("industry"),
108
- "all_industries": ", ".join([i.get("name") for i in a.get("industries", [])]) if a.get("industries") else None,
109
- "first_visit_date": a.get("first_visit_date"),
110
  "last_visit_date": a.get("last_visit_date"),
111
  "total_visits": a.get("visits"),
112
  "lead_quality_score": a.get("quality"),
113
  "revenue": a.get("revenue"),
114
  "employee_count": a.get("employee_count"),
115
- "employees_min": a.get("employees_range", {}).get("min") if a.get("employees_range") else None,
116
- "employees_max": a.get("employees_range", {}).get("max") if a.get("employees_range") else None,
117
  "assignee": a.get("assignee"),
118
- "emailed_to": a.get("emailed_to"),
119
- "crm_lead_id": a.get("crm_lead_id"),
120
  "crm_organization_id": a.get("crm_organization_id"),
121
- "tags": ", ".join(a.get("tags", [])) if a.get("tags") else None,
122
  "linkedin_url": a.get("linkedin_url"),
123
- "twitter_handle": a.get("twitter_handle"),
124
- "facebook_url": a.get("facebook_url"),
125
  "country": loc.get("country"),
126
- "region": loc.get("region"),
127
  "city": loc.get("city"),
128
- "leadfeeder_url": a.get("view_in_leadfeeder"),
129
  "landing_page_path": None,
130
  "exit_page_path": None
131
  })
132
-
133
- print(f"✅ Page {page} loaded. Rows: {len(rows)}")
134
  page += 1
135
-
136
  except Exception as e:
137
- print(f"Error on page {page}: {e}")
138
  break
139
-
140
  df = pd.DataFrame(rows)
141
  if not df.empty:
142
  df["last_visit_date"] = pd.to_datetime(df["last_visit_date"], errors="coerce")
143
  return df
144
 
145
- def enrich_leads_with_visits(df, start_date, end_date, max_rows=None, progress=gr.Progress()):
146
- """
147
- Step 2: Deep enrich with visit_route parsing for Exit Page.
148
- """
149
- if df.empty:
150
- return df
151
-
152
  target_df = df.head(max_rows) if max_rows else df
153
- total = len(target_df)
154
-
155
- print(f"🕵️ Deep enriching {total} rows ({start_date} to {end_date})...")
156
 
157
  for index, row in target_df.iterrows():
158
- if row.get("landing_page_path") and row.get("exit_page_path"):
159
- continue
160
-
161
  lead_id = row["lead_id"]
162
  try:
163
  visit_data = make_request(
164
  f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads/{lead_id}/visits",
165
- params={
166
- "start_date": start_date,
167
- "end_date": end_date,
168
- "page[size]": 1,
169
- "include": "page_views"
170
- }
171
  )
172
-
173
  visits = visit_data.get("data", [])
174
  included = visit_data.get("included", [])
175
- landing = None
176
- exit_p = None
177
 
178
  if visits:
179
  visit = visits[0]
180
  v_attrs = visit.get("attributes", {})
181
-
182
- # A. Direct Attributes
183
  landing = v_attrs.get("landing_page_path") or v_attrs.get("landing_page_url")
184
 
185
- # B. Parse 'visit_route'
186
  visit_route = v_attrs.get("visit_route", [])
187
  if visit_route and isinstance(visit_route, list):
188
- last_step = visit_route[-1]
189
- if not exit_p: exit_p = last_step.get("page_path") or last_step.get("page_url")
190
-
191
- if not landing:
192
- first_step = visit_route[0]
193
- landing = first_step.get("page_path") or first_step.get("page_url")
194
-
195
- # C. Deep Link via 'included' Page Views
196
  if not landing or not exit_p:
197
  pv_map = {p["id"]: p["attributes"] for p in included if p["type"] == "page_views"}
198
  pv_ids = [r["id"] for r in visit.get("relationships", {}).get("page_views", {}).get("data", [])]
199
-
200
  if pv_ids:
201
- if not landing:
202
- first_pv = pv_map.get(pv_ids[0])
203
- if first_pv: landing = first_pv.get("url") or first_pv.get("path")
204
- if not exit_p:
205
- last_pv = pv_map.get(pv_ids[-1])
206
- if last_pv: exit_p = last_pv.get("url") or last_pv.get("path")
207
-
208
  df.at[index, "landing_page_path"] = landing
209
  df.at[index, "exit_page_path"] = exit_p
210
- except Exception as e:
211
- print(f"Failed to enrich lead {lead_id}: {e}")
212
-
213
- if max_rows and index % 5 == 0:
214
- progress(index / total, desc="Enriching...")
215
-
216
  return df
217
 
218
  # =====================================================
219
- # DEBUGGER
220
  # =====================================================
221
- def inspect_raw_json(lead_id, start_date, end_date):
222
- if not lead_id: return "Please enter a Lead ID"
223
- try:
224
- url = f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads/{lead_id}/visits"
225
- params = {
226
- "start_date": start_date,
227
- "end_date": end_date,
228
- "page[size]": 1,
229
- "include": "page_views"
230
- }
231
- r = requests.get(url, headers=HEADERS, params=params)
232
- return json.dumps(r.json(), indent=2)
233
- except Exception as e:
234
- return str(e)
235
 
236
- # =====================================================
237
- # DATA WRAPPERS
238
- # =====================================================
239
- def load_preview(start, end):
240
- df = fetch_basic_leads(start, end)
241
- if df.empty: return df, pd.DataFrame(), "⚠️ No data found."
242
-
243
- df_preview = df.copy()
244
- df_preview = enrich_leads_with_visits(df_preview, start, end, max_rows=10)
245
-
246
- status = f"✅ Loaded **{len(df):,} companies**. Preview shows top 10 rows enriched."
247
- return df, df_preview.head(10), status
248
-
249
- def download_full_excel(df, start, end):
250
- if df is None or df.empty: return None
251
- print("⏳ Starting full enrichment for Excel export...")
252
- enriched_df = enrich_leads_with_visits(df.copy(), start, end)
253
- path = "/tmp/leadfeeder_company_data_full.xlsx"
254
- enriched_df.to_excel(path, index=False)
255
- return path
256
-
257
- # =====================================================
258
- # CHART ENGINES
259
- # =====================================================
260
- def build_kpis(df):
261
- if df is None or df.empty: return 0, 0, 0, 0, 0, 0
262
- return (
263
- len(df),
264
- df["total_visits"].gt(0).sum(),
265
- int(df["total_visits"].sum()),
266
- round(df["lead_quality_score"].mean(), 2),
267
- round(df["crm_organization_id"].notna().mean() * 100, 1),
268
- round(df["linkedin_url"].notna().mean() * 100, 1),
269
- )
270
-
271
- def build_dashboard(df, preset, top_n):
272
- if df is None or df.empty: return px.bar(title="No Data")
273
  dim, metric, agg = DASHBOARD_PRESETS[preset]
 
274
  if agg == "count":
275
  grouped = df.groupby(dim, dropna=False).size().reset_index(name="value")
276
  else:
277
  grouped = df.groupby(dim, dropna=False)[metric].agg(agg).reset_index(name="value")
278
- return px.bar(grouped.sort_values("value", ascending=False).head(top_n), x=dim, y="value", title=preset)
 
 
 
 
 
 
 
 
279
 
280
- def get_trend_filter_options(df, preset):
281
- if df is None or df.empty:
282
- return gr.update(choices=[], value=None, visible=False)
283
- metric, agg, segment = TREND_PRESETS[preset]
284
- if not segment:
285
- return gr.update(choices=[], value=None, visible=False)
286
- options = sorted(df[segment].dropna().unique().tolist())
287
- top_5 = df[segment].value_counts().head(5).index.tolist()
288
- return gr.update(choices=options, value=top_5, visible=True, label=f"Filter {segment}")
289
-
290
- def build_trend(df, preset, grain, filter_values):
291
- if df is None or df.empty: return px.line(title="No Data Loaded")
292
  metric, agg, segment = TREND_PRESETS[preset]
293
- df = df.dropna(subset=["last_visit_date"]).copy()
 
 
 
 
 
 
 
 
 
 
 
294
 
295
- if segment and filter_values:
296
- df = df[df[segment].isin(filter_values)]
297
- if df.empty: return px.line(title="No data for selected filters")
298
-
299
- if grain == "Weekly":
300
- df["period"] = df["last_visit_date"].dt.to_period("W").astype(str)
301
- elif grain == "Monthly":
302
- df["period"] = df["last_visit_date"].dt.to_period("M").astype(str)
303
  else:
304
- df["period"] = df["last_visit_date"].dt.date
 
305
 
306
- val = metric
307
- if agg == "count":
308
- df["_v"] = df[metric].notna().astype(int)
309
- val = "_v"
 
 
 
 
 
 
 
 
 
 
 
 
 
310
 
311
- if segment:
312
- ts = df.groupby(["period", segment])[val].agg(agg).reset_index()
313
- title = f"{preset} (Filtered)" if filter_values else preset
314
- return px.line(ts, x="period", y=val, color=segment, template="plotly_white", title=title)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
315
 
316
- ts = df.groupby("period")[val].agg(agg).reset_index()
317
- return px.line(ts, x="period", y=val, markers=True, template="plotly_white", title=preset)
 
 
 
 
 
 
 
 
318
 
319
  # =====================================================
320
- # UI LAYOUT
321
  # =====================================================
322
- with gr.Blocks(title="Leadfeeder Pro Analytics") as demo:
323
- gr.Markdown("## 🚀 Leadfeeder Analytics")
324
-
325
- with gr.Row():
326
- pwd = gr.Textbox(type="password", label="App Password")
327
- gr.Button("Auth").click(lambda p: gr.Info("Success") if p==APP_PASSWORD else gr.Error("Invalid"), pwd, None)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
328
 
329
- df_state = gr.State()
330
- status = gr.Markdown()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
331
 
332
- with gr.Tabs():
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
333
  # --- TAB 1: DATA ---
334
- with gr.Tab("📋 Data & Report"):
335
- with gr.Row():
336
- start = gr.Textbox(label="Start Date", value=(date.today()-timedelta(days=30)).isoformat())
337
- end = gr.Textbox(label="End Date", value=date.today().isoformat())
338
-
339
- with gr.Row():
340
- btn_load = gr.Button("1. Load Data (Preview)", variant="primary")
341
- btn_dl = gr.Button("2. Enrich & Download Full Excel")
342
- file_dl = gr.File(label="Download Excel")
343
-
344
- table = gr.Dataframe(label="Preview (Top 10 Enriched)", interactive=True)
345
-
346
- btn_load.click(load_preview, [start, end], [df_state, table, status])
347
- btn_dl.click(download_full_excel, [df_state, start, end], file_dl)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
348
 
349
  # --- TAB 2: DASHBOARD ---
350
- with gr.Tab("📊 Dashboard"):
351
- kpis = [gr.Number(label=l) for l in ["Total Companies", "Active Companies", "Total Visits", "Avg Quality", "CRM %", "LinkedIn %"]]
352
- gr.Button("Refresh KPIs").click(build_kpis, df_state, kpis)
353
-
354
- with gr.Row():
355
- preset = gr.Dropdown(choices=list(DASHBOARD_PRESETS.keys()), label="Chart")
356
- top_n = gr.Slider(5, 50, value=10, label="Top N")
357
 
358
- chart = gr.Plot()
359
- gr.Button("Build View").click(build_dashboard, [df_state, preset, top_n], chart)
360
 
361
  # --- TAB 3: TRENDS ---
362
- with gr.Tab("📈 Trends"):
363
- with gr.Row():
364
- trend_view = gr.Dropdown(choices=list(TREND_PRESETS.keys()), label="Select Trend")
365
- grain = gr.Radio(["Daily", "Weekly", "Monthly"], value="Daily", label="Granularity")
 
 
366
 
367
- filter_dropdown = gr.Dropdown(multiselect=True, visible=False, label="Filter Segments")
368
- plot = gr.Plot()
 
 
 
 
 
 
 
 
 
 
 
 
369
 
370
- trend_view.change(get_trend_filter_options, [df_state, trend_view], filter_dropdown)
371
- gr.Button("Show Trend", variant="primary").click(
372
- build_trend,
373
- [df_state, trend_view, grain, filter_dropdown],
374
- plot
375
- )
376
 
377
- # --- TAB 4: DEBUGGER ---
378
- with gr.Tab("🛠️ Debugger"):
379
- gr.Markdown("Inspect raw API data for a specific lead.")
380
- dbg_id = gr.Textbox(label="Lead ID")
381
- dbg_btn = gr.Button("Inspect Raw JSON")
382
- dbg_out = gr.Code(language="json")
383
- dbg_btn.click(inspect_raw_json, [dbg_id, start, end], dbg_out)
384
 
385
- if __name__ == "__main__":
386
- demo.launch(server_name="0.0.0.0", server_port=7860)
 
3
  import requests
4
  import json
5
  import pandas as pd
6
+ import plotly.graph_objects as go
7
  import plotly.express as px
8
  from datetime import date, timedelta
9
+ from nicegui import ui, app, run
10
 
11
  # =====================================================
12
  # CONFIG / SECRETS
 
15
  APP_PASSWORD = os.getenv("APP_PASSWORD")
16
 
17
  if not API_TOKEN or not APP_PASSWORD:
18
+ # Fallback for testing if env vars aren't set
19
+ print("⚠️ WARNING: Env vars not set. Using placeholders for code structure.")
20
+ API_TOKEN = "PLACEHOLDER_TOKEN"
21
+ APP_PASSWORD = "password123"
22
 
23
+ ACCOUNT_ID = "255333"
24
  BASE_URL = "https://api.leadfeeder.com"
25
+ PAGE_SIZE = 50
26
 
27
  HEADERS = {
28
  "Authorization": f"Token token={API_TOKEN}",
 
30
  }
31
 
32
  # =====================================================
33
+ # LOGIC & PRESETS (Pure Python Functions)
34
  # =====================================================
35
  DASHBOARD_PRESETS = {
36
  "Total Visits by Industry": ("primary_industry", "total_visits", "sum"),
 
52
  "Visits Trend by Assignee": ("total_visits", "sum", "assignee"),
53
  }
54
 
 
 
 
55
  def make_request(url, params=None):
56
  retries = 3
57
  while retries > 0:
58
  r = requests.get(url, headers=HEADERS, params=params, timeout=45)
59
  if r.status_code == 429:
 
60
  time.sleep(61)
61
  retries -= 1
62
  continue
 
64
  return r.json()
65
  raise Exception("Max retries exceeded")
66
 
 
 
 
67
  def fetch_basic_leads(start_date, end_date):
 
68
  page = 1
69
  rows = []
 
 
 
70
  while True:
71
  try:
72
  js = make_request(
73
  f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads",
74
+ params={"start_date": start_date, "end_date": end_date, "page[number]": page, "page[size]": PAGE_SIZE, "include": "location"}
 
 
 
 
 
 
75
  )
76
  data = js.get("data", [])
77
+ if not data: break
78
+
 
79
  included = js.get("included", [])
80
  loc_map = {str(i["id"]): i["attributes"] for i in included if i["type"] == "locations"}
81
 
 
83
  a = lead["attributes"]
84
  loc_id = lead.get("relationships", {}).get("location", {}).get("data", {}).get("id")
85
  loc = loc_map.get(str(loc_id), {})
 
86
  rows.append({
87
  "lead_id": lead.get("id"),
88
  "company_name": a.get("name"),
89
  "website_url": a.get("website_url"),
 
 
90
  "primary_industry": a.get("industry"),
 
 
91
  "last_visit_date": a.get("last_visit_date"),
92
  "total_visits": a.get("visits"),
93
  "lead_quality_score": a.get("quality"),
94
  "revenue": a.get("revenue"),
95
  "employee_count": a.get("employee_count"),
 
 
96
  "assignee": a.get("assignee"),
 
 
97
  "crm_organization_id": a.get("crm_organization_id"),
 
98
  "linkedin_url": a.get("linkedin_url"),
 
 
99
  "country": loc.get("country"),
 
100
  "city": loc.get("city"),
 
101
  "landing_page_path": None,
102
  "exit_page_path": None
103
  })
 
 
104
  page += 1
 
105
  except Exception as e:
106
+ print(f"Error: {e}")
107
  break
108
+
109
  df = pd.DataFrame(rows)
110
  if not df.empty:
111
  df["last_visit_date"] = pd.to_datetime(df["last_visit_date"], errors="coerce")
112
  return df
113
 
114
+ def enrich_leads_with_visits(df, start_date, end_date, max_rows=None):
115
+ if df.empty: return df
 
 
 
 
 
116
  target_df = df.head(max_rows) if max_rows else df
 
 
 
117
 
118
  for index, row in target_df.iterrows():
119
+ if row.get("landing_page_path") and row.get("exit_page_path"): continue
120
+
 
121
  lead_id = row["lead_id"]
122
  try:
123
  visit_data = make_request(
124
  f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads/{lead_id}/visits",
125
+ params={"start_date": start_date, "end_date": end_date, "page[size]": 1, "include": "page_views"}
 
 
 
 
 
126
  )
 
127
  visits = visit_data.get("data", [])
128
  included = visit_data.get("included", [])
129
+ landing, exit_p = None, None
 
130
 
131
  if visits:
132
  visit = visits[0]
133
  v_attrs = visit.get("attributes", {})
 
 
134
  landing = v_attrs.get("landing_page_path") or v_attrs.get("landing_page_url")
135
 
136
+ # Visit Route Logic
137
  visit_route = v_attrs.get("visit_route", [])
138
  if visit_route and isinstance(visit_route, list):
139
+ if not exit_p: exit_p = visit_route[-1].get("page_path") or visit_route[-1].get("page_url")
140
+ if not landing: landing = visit_route[0].get("page_path") or visit_route[0].get("page_url")
141
+
142
+ # Page View Logic
 
 
 
 
143
  if not landing or not exit_p:
144
  pv_map = {p["id"]: p["attributes"] for p in included if p["type"] == "page_views"}
145
  pv_ids = [r["id"] for r in visit.get("relationships", {}).get("page_views", {}).get("data", [])]
 
146
  if pv_ids:
147
+ if not landing: landing = pv_map.get(pv_ids[0], {}).get("url")
148
+ if not exit_p: exit_p = pv_map.get(pv_ids[-1], {}).get("url")
149
+
 
 
 
 
150
  df.at[index, "landing_page_path"] = landing
151
  df.at[index, "exit_page_path"] = exit_p
152
+ except: pass
 
 
 
 
 
153
  return df
154
 
155
  # =====================================================
156
+ # CHART BUILDERS (Plotly GO)
157
  # =====================================================
158
+ def create_empty_figure(text="No Data Loaded"):
159
+ fig = go.Figure()
160
+ fig.update_layout(title=text, template="plotly_white")
161
+ return fig
 
 
 
 
 
 
 
 
 
 
162
 
163
+ def build_dashboard_chart(df, preset):
164
+ if df is None or df.empty: return create_empty_figure()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
165
  dim, metric, agg = DASHBOARD_PRESETS[preset]
166
+
167
  if agg == "count":
168
  grouped = df.groupby(dim, dropna=False).size().reset_index(name="value")
169
  else:
170
  grouped = df.groupby(dim, dropna=False)[metric].agg(agg).reset_index(name="value")
171
+
172
+ grouped = grouped.sort_values("value", ascending=False).head(15)
173
+
174
+ fig = go.Figure(go.Bar(
175
+ x=grouped[dim], y=grouped["value"],
176
+ marker_color='#5898d4', text=grouped["value"], textposition='auto'
177
+ ))
178
+ fig.update_layout(title=f"<b>{preset}</b>", template="plotly_white", margin=dict(l=20,r=20,t=50,b=20))
179
+ return fig
180
 
181
+ def build_trend_chart(df, preset, grain, filters):
182
+ if df is None or df.empty: return create_empty_figure()
 
 
 
 
 
 
 
 
 
 
183
  metric, agg, segment = TREND_PRESETS[preset]
184
+ df_c = df.dropna(subset=["last_visit_date"]).copy()
185
+
186
+ if segment and filters:
187
+ df_c = df_c[df_c[segment].isin(filters)]
188
+ if df_c.empty: return create_empty_figure("No data for filters")
189
+
190
+ if grain == "Weekly": df_c["period"] = df_c["last_visit_date"].dt.to_period("W").astype(str)
191
+ elif grain == "Monthly": df_c["period"] = df_c["last_visit_date"].dt.to_period("M").astype(str)
192
+ else: df_c["period"] = df_c["last_visit_date"].dt.date
193
+
194
+ val = "_v" if agg == "count" else metric
195
+ if agg == "count": df_c["_v"] = df_c[metric].notna().astype(int)
196
 
197
+ fig = go.Figure()
198
+ if segment:
199
+ ts = df_c.groupby(["period", segment])[val].agg(agg).reset_index()
200
+ for seg in ts[segment].unique():
201
+ seg_data = ts[ts[segment] == seg]
202
+ fig.add_trace(go.Scatter(x=seg_data["period"], y=seg_data[val], mode='lines+markers', name=str(seg)))
 
 
203
  else:
204
+ ts = df_c.groupby("period")[val].agg(agg).reset_index()
205
+ fig.add_trace(go.Scatter(x=ts["period"], y=ts[val], mode='lines+markers', name="Total", line=dict(color='#00CC96', width=3)))
206
 
207
+ fig.update_layout(title=f"<b>{preset}</b>", template="plotly_white")
208
+ return fig
209
+
210
+ def build_advanced_charts(df, search_lead, search_comp, start, end):
211
+ if df is None or df.empty: return create_empty_figure(), create_empty_figure(), create_empty_figure()
212
+
213
+ filtered_df = df.copy()
214
+ is_search = False
215
+
216
+ if search_lead:
217
+ filtered_df = filtered_df[filtered_df["lead_id"] == search_lead.strip()]
218
+ is_search = True
219
+ elif search_comp:
220
+ filtered_df = filtered_df[filtered_df["company_name"].str.contains(search_comp, case=False, na=False)]
221
+ is_search = True
222
+ else:
223
+ filtered_df = filtered_df.sort_values("total_visits", ascending=False).head(50)
224
 
225
+ # Auto Enrich
226
+ missing = filtered_df[filtered_df["landing_page_path"].isnull()]
227
+ if not missing.empty:
228
+ enriched = enrich_leads_with_visits(missing, start, end)
229
+ filtered_df.update(enriched)
230
+
231
+ # Sankey
232
+ flow_df = filtered_df.dropna(subset=["landing_page_path", "exit_page_path"])
233
+ if flow_df.empty:
234
+ sankey = create_empty_figure("No Flow Data")
235
+ else:
236
+ counts = flow_df.groupby(["landing_page_path", "exit_page_path"]).size().reset_index(name="count")
237
+ if not is_search: counts = counts.sort_values("count", ascending=False).head(15)
238
+
239
+ all_nodes = list(pd.concat([counts["landing_page_path"], counts["exit_page_path"]]).unique())
240
+ node_map = {node: i for i, node in enumerate(all_nodes)}
241
+
242
+ sankey = go.Figure(go.Sankey(
243
+ node=dict(pad=15, thickness=20, line=dict(color="black", width=0.5), label=all_nodes, color="blue"),
244
+ link=dict(source=counts["landing_page_path"].map(node_map), target=counts["exit_page_path"].map(node_map), value=counts["count"])
245
+ ))
246
+ sankey.update_layout(title="<b>User Flow: Landing → Exit</b>", font_size=10)
247
+
248
+ # Heatmap
249
+ temp_df = filtered_df.dropna(subset=["last_visit_date"]).copy()
250
+ if temp_df.empty:
251
+ heatmap = create_empty_figure("No Date Data")
252
+ else:
253
+ temp_df["day"] = temp_df["last_visit_date"].dt.day_name()
254
+ temp_df["hour"] = temp_df["last_visit_date"].dt.hour
255
+ days_order = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
256
+ hm_data = temp_df.groupby(["day", "hour"]).size().reset_index(name="visits")
257
+ hm_pivot = hm_data.pivot(index="day", columns="hour", values="visits").fillna(0).reindex(days_order)
258
+
259
+ heatmap = go.Figure(go.Heatmap(z=hm_pivot.values, x=hm_pivot.columns, y=hm_pivot.index, colorscale='Viridis'))
260
+ heatmap.update_layout(title="<b>Activity Heatmap</b>")
261
+
262
+ # Matrix (Aggregate)
263
+ roi_df = df.groupby("primary_industry").agg({"total_visits": "sum", "lead_quality_score": "mean", "company_name": "count"}).reset_index()
264
+ roi_df = roi_df[roi_df["company_name"] > 2]
265
 
266
+ if roi_df.empty:
267
+ matrix = create_empty_figure("Not enough data")
268
+ else:
269
+ matrix = go.Figure(go.Scatter(
270
+ x=roi_df["total_visits"], y=roi_df["lead_quality_score"], mode='markers', text=roi_df["primary_industry"],
271
+ marker=dict(size=roi_df["company_name"], sizemode='area', sizeref=2.*max(roi_df["company_name"])/(40.**2), sizemin=4, color=roi_df["lead_quality_score"], colorscale='RdYlGn', showscale=True)
272
+ ))
273
+ matrix.update_layout(title="<b>Industry Matrix</b>", xaxis_title="Volume", yaxis_title="Quality", template="plotly_white")
274
+
275
+ return sankey, heatmap, matrix
276
 
277
  # =====================================================
278
+ # UI APPLICATION (NiceGUI)
279
  # =====================================================
280
+ class LeadfeederApp:
281
+ def __init__(self):
282
+ self.df = pd.DataFrame()
283
+ self.start_date_val = (date.today() - timedelta(days=30)).isoformat()
284
+ self.end_date_val = date.today().isoformat()
285
+
286
+ async def load_data(self):
287
+ if self.pwd_input.value != APP_PASSWORD:
288
+ ui.notify('Invalid Password!', type='negative')
289
+ return
290
+
291
+ self.loading_spinner.visible = True
292
+ ui.notify('Fetching data... this may take a moment.')
293
+
294
+ # Run heavy fetch in background
295
+ self.df = await run.io_bound(fetch_basic_leads, self.date_start.value, self.date_end.value)
296
+
297
+ if self.df.empty:
298
+ ui.notify('No data found.', type='warning')
299
+ else:
300
+ # Enrich top 10 for preview immediately
301
+ preview_df = self.df.copy()
302
+ preview_df = await run.io_bound(enrich_leads_with_visits, preview_df, self.date_start.value, self.date_end.value, 10)
303
+
304
+ # Update UI
305
+ self.update_kpis()
306
+ self.update_aggrid(preview_df.head(10))
307
+ self.refresh_dashboard()
308
+ self.refresh_trends()
309
+ ui.notify(f'Loaded {len(self.df)} companies successfully!', type='positive')
310
+
311
+ self.loading_spinner.visible = False
312
+
313
+ def update_kpis(self):
314
+ if self.df.empty: return
315
+ self.kpi_total.value = len(self.df)
316
+ self.kpi_visits.value = int(self.df["total_visits"].sum())
317
+ self.kpi_quality.value = round(self.df["lead_quality_score"].mean(), 2)
318
+ self.kpi_crm.value = f"{round(self.df['crm_organization_id'].notna().mean() * 100, 1)}%"
319
+
320
+ def update_aggrid(self, df_show):
321
+ # Convert timestamp to string for display
322
+ display_df = df_show.copy()
323
+ for col in display_df.select_dtypes(include=['datetime64']).columns:
324
+ display_df[col] = display_df[col].astype(str)
325
+ self.grid.options['rowData'] = display_df.to_dict(orient='records')
326
+ self.grid.update()
327
+
328
+ async def download_full_report(self):
329
+ if self.df.empty: return
330
+ ui.notify('Generating Excel... please wait.')
331
+
332
+ # Run full enrichment
333
+ enriched_df = await run.io_bound(enrich_leads_with_visits, self.df.copy(), self.date_start.value, self.date_end.value)
334
+
335
+ path = f"/tmp/leadfeeder_report_{int(time.time())}.xlsx"
336
+ enriched_df.to_excel(path, index=False)
337
+ ui.download(path)
338
+ ui.notify('Download started!', type='positive')
339
+
340
+ def refresh_dashboard(self):
341
+ fig = build_dashboard_chart(self.df, self.dash_preset.value)
342
+ self.dash_plot.update_figure(fig)
343
+
344
+ def update_trend_options(self):
345
+ if self.df.empty: return
346
+ _, _, segment = TREND_PRESETS[self.trend_preset.value]
347
+ if segment:
348
+ opts = sorted(self.df[segment].dropna().unique().tolist())
349
+ self.trend_filter.options = opts
350
+ self.trend_filter.visible = True
351
+ else:
352
+ self.trend_filter.visible = False
353
+ self.trend_filter.value = []
354
+
355
+ def refresh_trends(self):
356
+ fig = build_trend_chart(self.df, self.trend_preset.value, self.trend_grain.value, self.trend_filter.value)
357
+ self.trend_plot.update_figure(fig)
358
+
359
+ async def run_advanced_analytics(self):
360
+ if self.df.empty: return
361
+ ui.notify('Analyzing user flows...')
362
+ sankey, heatmap, matrix = await run.io_bound(
363
+ build_advanced_charts, self.df, self.adv_lead_id.value, self.adv_comp.value,
364
+ self.date_start.value, self.date_end.value
365
+ )
366
+ self.sankey_plot.update_figure(sankey)
367
+ self.heatmap_plot.update_figure(heatmap)
368
+ self.matrix_plot.update_figure(matrix)
369
 
370
+ async def run_debugger(self):
371
+ if not self.debug_id.value: return
372
+ res = await run.io_bound(
373
+ requests.get,
374
+ f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads/{self.debug_id.value}/visits",
375
+ headers=HEADERS,
376
+ params={"start_date": self.date_start.value, "end_date": self.date_end.value, "page[size]": 1, "include": "page_views"}
377
+ )
378
+ self.debug_output.set_content(f"```json\n{json.dumps(res.json(), indent=2)}\n```")
379
+
380
+ # =====================================================
381
+ # UI LAYOUT
382
+ # =====================================================
383
+ @ui.page('/')
384
+ def main_page():
385
+ app_logic = LeadfeederApp()
386
 
387
+ # --- HEADER ---
388
+ with ui.header().classes(replace='row items-center') as header:
389
+ ui.icon('analytics', size='32px').classes('text-white q-mr-sm')
390
+ ui.label('Leadfeeder Pro Analytics').classes('text-h6 text-white')
391
+ ui.space()
392
+ app_logic.pwd_input = ui.input('App Password', password=True).classes('bg-white rounded q-px-sm')
393
+
394
+ # --- TABS ---
395
+ with ui.tabs().classes('w-full') as tabs:
396
+ data_tab = ui.tab('Data & Report')
397
+ dash_tab = ui.tab('Dashboard')
398
+ trend_tab = ui.tab('Trends')
399
+ adv_tab = ui.tab('Advanced Analytics')
400
+ debug_tab = ui.tab('Debugger')
401
+
402
+ with ui.tab_panels(tabs, value=data_tab).classes('w-full q-pa-md'):
403
+
404
  # --- TAB 1: DATA ---
405
+ with ui.tab_panel(data_tab):
406
+ with ui.row().classes('w-full items-center gap-4'):
407
+ app_logic.date_start = ui.input('Start Date', value=app_logic.start_date_val)
408
+ app_logic.date_end = ui.input('End Date', value=app_logic.end_date_val)
409
+ ui.button('Load Data', on_click=app_logic.load_data, icon='cloud_download')
410
+ ui.button('Download Excel (Full)', on_click=app_logic.download_full_report, icon='file_download').classes('bg-green')
411
+ app_logic.loading_spinner = ui.spinner(size='lg').props('color=primary').bind_visibility_from(app_logic.loading_spinner, 'visible')
412
+ app_logic.loading_spinner.visible = False
413
+
414
+ # KPIS
415
+ with ui.row().classes('w-full gap-4 q-my-md'):
416
+ with ui.card().classes('w-48'):
417
+ ui.label('Total Companies').classes('text-caption')
418
+ app_logic.kpi_total = ui.label('0').classes('text-h5')
419
+ with ui.card().classes('w-48'):
420
+ ui.label('Total Visits').classes('text-caption')
421
+ app_logic.kpi_visits = ui.label('0').classes('text-h5')
422
+ with ui.card().classes('w-48'):
423
+ ui.label('Avg Quality').classes('text-caption')
424
+ app_logic.kpi_quality = ui.label('0').classes('text-h5')
425
+ with ui.card().classes('w-48'):
426
+ ui.label('CRM Coverage').classes('text-caption')
427
+ app_logic.kpi_crm = ui.label('0%').classes('text-h5')
428
+
429
+ # PREVIEW GRID
430
+ ui.label('Data Preview (Top 10 Enriched)').classes('text-h6 q-mt-md')
431
+ app_logic.grid = ui.aggrid({
432
+ 'columnDefs': [
433
+ {'headerName': 'Company', 'field': 'company_name'},
434
+ {'headerName': 'Industry', 'field': 'primary_industry'},
435
+ {'headerName': 'Visits', 'field': 'total_visits'},
436
+ {'headerName': 'Landing Page', 'field': 'landing_page_path'},
437
+ {'headerName': 'Exit Page', 'field': 'exit_page_path'},
438
+ ],
439
+ 'rowData': []
440
+ }).classes('h-96 w-full')
441
 
442
  # --- TAB 2: DASHBOARD ---
443
+ with ui.tab_panel(dash_tab):
444
+ with ui.row().classes('items-center'):
445
+ app_logic.dash_preset = ui.select(list(DASHBOARD_PRESETS.keys()), label="Select Chart", value="Total Visits by Industry", on_change=app_logic.refresh_dashboard).classes('w-64')
446
+ ui.button('Refresh', icon='refresh', on_click=app_logic.refresh_dashboard).props('flat')
 
 
 
447
 
448
+ app_logic.dash_plot = ui.plotly({}).classes('w-full h-128')
 
449
 
450
  # --- TAB 3: TRENDS ---
451
+ with ui.tab_panel(trend_tab):
452
+ with ui.row().classes('items-start gap-4'):
453
+ app_logic.trend_preset = ui.select(list(TREND_PRESETS.keys()), label="Metric", value="Total Visits Trend", on_change=lambda: [app_logic.update_trend_options(), app_logic.refresh_trends()]).classes('w-64')
454
+ app_logic.trend_grain = ui.radio(['Daily', 'Weekly', 'Monthly'], value='Daily', on_change=app_logic.refresh_trends).props('inline')
455
+ app_logic.trend_filter = ui.select([], multiple=True, label="Filter Segments", on_change=app_logic.refresh_trends).classes('w-96').props('use-chips')
456
+ app_logic.trend_filter.visible = False
457
 
458
+ app_logic.trend_plot = ui.plotly({}).classes('w-full h-128')
459
+
460
+ # --- TAB 4: ADVANCED ---
461
+ with ui.tab_panel(adv_tab):
462
+ with ui.row().classes('items-center gap-4'):
463
+ app_logic.adv_lead_id = ui.input('Search Lead ID')
464
+ app_logic.adv_comp = ui.input('Search Company Name')
465
+ ui.button('Analyze Flow', on_click=app_logic.run_advanced_analytics, icon='insights')
466
+
467
+ with ui.row().classes('w-full'):
468
+ with ui.column().classes('w-1/2'):
469
+ app_logic.sankey_plot = ui.plotly({}).classes('w-full h-96')
470
+ with ui.column().classes('w-1/2'):
471
+ app_logic.heatmap_plot = ui.plotly({}).classes('w-full h-96')
472
 
473
+ app_logic.matrix_plot = ui.plotly({}).classes('w-full h-96 q-mt-md')
 
 
 
 
 
474
 
475
+ # --- TAB 5: DEBUGGER ---
476
+ with ui.tab_panel(debug_tab):
477
+ with ui.row():
478
+ app_logic.debug_id = ui.input('Lead ID')
479
+ ui.button('Inspect JSON', on_click=app_logic.run_debugger)
480
+ app_logic.debug_output = ui.markdown('Raw JSON will appear here...')
 
481
 
482
+ ui.run(title='Leadfeeder Analytics', port=8080, favicon='🚀')