mktgtech commited on
Commit
47859aa
Β·
verified Β·
1 Parent(s): 1d750ec

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +231 -327
app.py CHANGED
@@ -3,10 +3,9 @@ import time
3
  import requests
4
  import json
5
  import pandas as pd
6
- import plotly.graph_objects as go
7
  import plotly.express as px
8
  from datetime import date, timedelta
9
- from nicegui import ui, app, run
10
 
11
  # =====================================================
12
  # CONFIG / SECRETS
@@ -15,14 +14,11 @@ API_TOKEN = os.getenv("LEADFEEDER_API_TOKEN")
15
  APP_PASSWORD = os.getenv("APP_PASSWORD")
16
 
17
  if not API_TOKEN or not APP_PASSWORD:
18
- # Fallback for testing if env vars aren't set
19
- print("⚠️ WARNING: Env vars not set. Using placeholders for code structure.")
20
- API_TOKEN = "PLACEHOLDER_TOKEN"
21
- APP_PASSWORD = "password123"
22
 
23
- ACCOUNT_ID = "255333"
24
  BASE_URL = "https://api.leadfeeder.com"
25
- PAGE_SIZE = 50
26
 
27
  HEADERS = {
28
  "Authorization": f"Token token={API_TOKEN}",
@@ -30,7 +26,7 @@ HEADERS = {
30
  }
31
 
32
  # =====================================================
33
- # LOGIC & PRESETS (Pure Python Functions)
34
  # =====================================================
35
  DASHBOARD_PRESETS = {
36
  "Total Visits by Industry": ("primary_industry", "total_visits", "sum"),
@@ -52,11 +48,15 @@ TREND_PRESETS = {
52
  "Visits Trend by Assignee": ("total_visits", "sum", "assignee"),
53
  }
54
 
 
 
 
55
  def make_request(url, params=None):
56
  retries = 3
57
  while retries > 0:
58
  r = requests.get(url, headers=HEADERS, params=params, timeout=45)
59
  if r.status_code == 429:
 
60
  time.sleep(61)
61
  retries -= 1
62
  continue
@@ -64,18 +64,32 @@ def make_request(url, params=None):
64
  return r.json()
65
  raise Exception("Max retries exceeded")
66
 
 
 
 
67
  def fetch_basic_leads(start_date, end_date):
 
68
  page = 1
69
  rows = []
 
 
 
70
  while True:
71
  try:
72
  js = make_request(
73
  f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads",
74
- params={"start_date": start_date, "end_date": end_date, "page[number]": page, "page[size]": PAGE_SIZE, "include": "location"}
 
 
 
 
 
 
75
  )
76
  data = js.get("data", [])
77
- if not data: break
78
-
 
79
  included = js.get("included", [])
80
  loc_map = {str(i["id"]): i["attributes"] for i in included if i["type"] == "locations"}
81
 
@@ -83,400 +97,290 @@ def fetch_basic_leads(start_date, end_date):
83
  a = lead["attributes"]
84
  loc_id = lead.get("relationships", {}).get("location", {}).get("data", {}).get("id")
85
  loc = loc_map.get(str(loc_id), {})
 
86
  rows.append({
87
  "lead_id": lead.get("id"),
88
  "company_name": a.get("name"),
89
  "website_url": a.get("website_url"),
 
 
90
  "primary_industry": a.get("industry"),
 
 
91
  "last_visit_date": a.get("last_visit_date"),
92
  "total_visits": a.get("visits"),
93
  "lead_quality_score": a.get("quality"),
94
  "revenue": a.get("revenue"),
95
  "employee_count": a.get("employee_count"),
 
 
96
  "assignee": a.get("assignee"),
 
 
97
  "crm_organization_id": a.get("crm_organization_id"),
 
98
  "linkedin_url": a.get("linkedin_url"),
 
 
99
  "country": loc.get("country"),
 
100
  "city": loc.get("city"),
 
101
  "landing_page_path": None,
102
  "exit_page_path": None
103
  })
 
 
104
  page += 1
 
105
  except Exception as e:
106
- print(f"Error: {e}")
107
  break
108
-
109
  df = pd.DataFrame(rows)
110
  if not df.empty:
111
  df["last_visit_date"] = pd.to_datetime(df["last_visit_date"], errors="coerce")
112
  return df
113
 
114
- def enrich_leads_with_visits(df, start_date, end_date, max_rows=None):
115
- if df.empty: return df
 
 
 
 
 
116
  target_df = df.head(max_rows) if max_rows else df
 
 
 
117
 
118
  for index, row in target_df.iterrows():
119
- if row.get("landing_page_path") and row.get("exit_page_path"): continue
120
-
 
121
  lead_id = row["lead_id"]
122
  try:
123
  visit_data = make_request(
124
  f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads/{lead_id}/visits",
125
- params={"start_date": start_date, "end_date": end_date, "page[size]": 1, "include": "page_views"}
 
 
 
 
 
126
  )
 
127
  visits = visit_data.get("data", [])
128
  included = visit_data.get("included", [])
129
- landing, exit_p = None, None
 
130
 
131
  if visits:
132
  visit = visits[0]
133
  v_attrs = visit.get("attributes", {})
 
 
134
  landing = v_attrs.get("landing_page_path") or v_attrs.get("landing_page_url")
135
 
136
- # Visit Route Logic
137
  visit_route = v_attrs.get("visit_route", [])
138
  if visit_route and isinstance(visit_route, list):
139
- if not exit_p: exit_p = visit_route[-1].get("page_path") or visit_route[-1].get("page_url")
140
- if not landing: landing = visit_route[0].get("page_path") or visit_route[0].get("page_url")
141
-
142
- # Page View Logic
 
 
 
 
143
  if not landing or not exit_p:
144
  pv_map = {p["id"]: p["attributes"] for p in included if p["type"] == "page_views"}
145
  pv_ids = [r["id"] for r in visit.get("relationships", {}).get("page_views", {}).get("data", [])]
 
146
  if pv_ids:
147
- if not landing: landing = pv_map.get(pv_ids[0], {}).get("url")
148
- if not exit_p: exit_p = pv_map.get(pv_ids[-1], {}).get("url")
149
-
 
 
 
 
150
  df.at[index, "landing_page_path"] = landing
151
  df.at[index, "exit_page_path"] = exit_p
152
- except: pass
 
 
 
 
 
153
  return df
154
 
155
  # =====================================================
156
- # CHART BUILDERS (Plotly GO)
157
  # =====================================================
158
- def create_empty_figure(text="No Data Loaded"):
159
- fig = go.Figure()
160
- fig.update_layout(title=text, template="plotly_white")
161
- return fig
 
 
 
 
 
 
 
 
 
 
162
 
163
- def build_dashboard_chart(df, preset):
164
- if df is None or df.empty: return create_empty_figure()
165
- dim, metric, agg = DASHBOARD_PRESETS[preset]
 
 
 
 
 
 
166
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
167
  if agg == "count":
168
  grouped = df.groupby(dim, dropna=False).size().reset_index(name="value")
169
  else:
170
  grouped = df.groupby(dim, dropna=False)[metric].agg(agg).reset_index(name="value")
171
-
172
- grouped = grouped.sort_values("value", ascending=False).head(15)
173
-
174
- fig = go.Figure(go.Bar(
175
- x=grouped[dim], y=grouped["value"],
176
- marker_color='#5898d4', text=grouped["value"], textposition='auto'
177
- ))
178
- fig.update_layout(title=f"<b>{preset}</b>", template="plotly_white", margin=dict(l=20,r=20,t=50,b=20))
179
- return fig
180
 
181
- def build_trend_chart(df, preset, grain, filters):
182
- if df is None or df.empty: return create_empty_figure()
 
183
  metric, agg, segment = TREND_PRESETS[preset]
184
- df_c = df.dropna(subset=["last_visit_date"]).copy()
185
-
186
- if segment and filters:
187
- df_c = df_c[df_c[segment].isin(filters)]
188
- if df_c.empty: return create_empty_figure("No data for filters")
189
-
190
- if grain == "Weekly": df_c["period"] = df_c["last_visit_date"].dt.to_period("W").astype(str)
191
- elif grain == "Monthly": df_c["period"] = df_c["last_visit_date"].dt.to_period("M").astype(str)
192
- else: df_c["period"] = df_c["last_visit_date"].dt.date
193
-
194
- val = "_v" if agg == "count" else metric
195
- if agg == "count": df_c["_v"] = df_c[metric].notna().astype(int)
196
-
197
- fig = go.Figure()
198
- if segment:
199
- ts = df_c.groupby(["period", segment])[val].agg(agg).reset_index()
200
- for seg in ts[segment].unique():
201
- seg_data = ts[ts[segment] == seg]
202
- fig.add_trace(go.Scatter(x=seg_data["period"], y=seg_data[val], mode='lines+markers', name=str(seg)))
203
- else:
204
- ts = df_c.groupby("period")[val].agg(agg).reset_index()
205
- fig.add_trace(go.Scatter(x=ts["period"], y=ts[val], mode='lines+markers', name="Total", line=dict(color='#00CC96', width=3)))
206
-
207
- fig.update_layout(title=f"<b>{preset}</b>", template="plotly_white")
208
- return fig
209
-
210
- def build_advanced_charts(df, search_lead, search_comp, start, end):
211
- if df is None or df.empty: return create_empty_figure(), create_empty_figure(), create_empty_figure()
212
-
213
- filtered_df = df.copy()
214
- is_search = False
215
 
216
- if search_lead:
217
- filtered_df = filtered_df[filtered_df["lead_id"] == search_lead.strip()]
218
- is_search = True
219
- elif search_comp:
220
- filtered_df = filtered_df[filtered_df["company_name"].str.contains(search_comp, case=False, na=False)]
221
- is_search = True
222
- else:
223
- filtered_df = filtered_df.sort_values("total_visits", ascending=False).head(50)
224
-
225
- # Auto Enrich
226
- missing = filtered_df[filtered_df["landing_page_path"].isnull()]
227
- if not missing.empty:
228
- enriched = enrich_leads_with_visits(missing, start, end)
229
- filtered_df.update(enriched)
230
-
231
- # Sankey
232
- flow_df = filtered_df.dropna(subset=["landing_page_path", "exit_page_path"])
233
- if flow_df.empty:
234
- sankey = create_empty_figure("No Flow Data")
235
- else:
236
- counts = flow_df.groupby(["landing_page_path", "exit_page_path"]).size().reset_index(name="count")
237
- if not is_search: counts = counts.sort_values("count", ascending=False).head(15)
238
-
239
- all_nodes = list(pd.concat([counts["landing_page_path"], counts["exit_page_path"]]).unique())
240
- node_map = {node: i for i, node in enumerate(all_nodes)}
241
-
242
- sankey = go.Figure(go.Sankey(
243
- node=dict(pad=15, thickness=20, line=dict(color="black", width=0.5), label=all_nodes, color="blue"),
244
- link=dict(source=counts["landing_page_path"].map(node_map), target=counts["exit_page_path"].map(node_map), value=counts["count"])
245
- ))
246
- sankey.update_layout(title="<b>User Flow: Landing β†’ Exit</b>", font_size=10)
247
-
248
- # Heatmap
249
- temp_df = filtered_df.dropna(subset=["last_visit_date"]).copy()
250
- if temp_df.empty:
251
- heatmap = create_empty_figure("No Date Data")
252
  else:
253
- temp_df["day"] = temp_df["last_visit_date"].dt.day_name()
254
- temp_df["hour"] = temp_df["last_visit_date"].dt.hour
255
- days_order = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
256
- hm_data = temp_df.groupby(["day", "hour"]).size().reset_index(name="visits")
257
- hm_pivot = hm_data.pivot(index="day", columns="hour", values="visits").fillna(0).reindex(days_order)
258
 
259
- heatmap = go.Figure(go.Heatmap(z=hm_pivot.values, x=hm_pivot.columns, y=hm_pivot.index, colorscale='Viridis'))
260
- heatmap.update_layout(title="<b>Activity Heatmap</b>")
261
-
262
- # Matrix (Aggregate)
263
- roi_df = df.groupby("primary_industry").agg({"total_visits": "sum", "lead_quality_score": "mean", "company_name": "count"}).reset_index()
264
- roi_df = roi_df[roi_df["company_name"] > 2]
265
-
266
- if roi_df.empty:
267
- matrix = create_empty_figure("Not enough data")
268
- else:
269
- matrix = go.Figure(go.Scatter(
270
- x=roi_df["total_visits"], y=roi_df["lead_quality_score"], mode='markers', text=roi_df["primary_industry"],
271
- marker=dict(size=roi_df["company_name"], sizemode='area', sizeref=2.*max(roi_df["company_name"])/(40.**2), sizemin=4, color=roi_df["lead_quality_score"], colorscale='RdYlGn', showscale=True)
272
- ))
273
- matrix.update_layout(title="<b>Industry Matrix</b>", xaxis_title="Volume", yaxis_title="Quality", template="plotly_white")
274
-
275
- return sankey, heatmap, matrix
276
-
277
- # =====================================================
278
- # UI APPLICATION (NiceGUI)
279
- # =====================================================
280
- class LeadfeederApp:
281
- def __init__(self):
282
- self.df = pd.DataFrame()
283
- self.start_date_val = (date.today() - timedelta(days=30)).isoformat()
284
- self.end_date_val = date.today().isoformat()
285
-
286
- async def load_data(self):
287
- if self.pwd_input.value != APP_PASSWORD:
288
- ui.notify('Invalid Password!', type='negative')
289
- return
290
-
291
- self.loading_spinner.visible = True
292
- ui.notify('Fetching data... this may take a moment.')
293
-
294
- # Run heavy fetch in background
295
- self.df = await run.io_bound(fetch_basic_leads, self.date_start.value, self.date_end.value)
296
-
297
- if self.df.empty:
298
- ui.notify('No data found.', type='warning')
299
- else:
300
- # Enrich top 10 for preview immediately
301
- preview_df = self.df.copy()
302
- preview_df = await run.io_bound(enrich_leads_with_visits, preview_df, self.date_start.value, self.date_end.value, 10)
303
-
304
- # Update UI
305
- self.update_kpis()
306
- self.update_aggrid(preview_df.head(10))
307
- self.refresh_dashboard()
308
- self.refresh_trends()
309
- ui.notify(f'Loaded {len(self.df)} companies successfully!', type='positive')
310
-
311
- self.loading_spinner.visible = False
312
-
313
- def update_kpis(self):
314
- if self.df.empty: return
315
- self.kpi_total.value = len(self.df)
316
- self.kpi_visits.value = int(self.df["total_visits"].sum())
317
- self.kpi_quality.value = round(self.df["lead_quality_score"].mean(), 2)
318
- self.kpi_crm.value = f"{round(self.df['crm_organization_id'].notna().mean() * 100, 1)}%"
319
-
320
- def update_aggrid(self, df_show):
321
- # Convert timestamp to string for display
322
- display_df = df_show.copy()
323
- for col in display_df.select_dtypes(include=['datetime64']).columns:
324
- display_df[col] = display_df[col].astype(str)
325
- self.grid.options['rowData'] = display_df.to_dict(orient='records')
326
- self.grid.update()
327
-
328
- async def download_full_report(self):
329
- if self.df.empty: return
330
- ui.notify('Generating Excel... please wait.')
331
-
332
- # Run full enrichment
333
- enriched_df = await run.io_bound(enrich_leads_with_visits, self.df.copy(), self.date_start.value, self.date_end.value)
334
 
335
- path = f"/tmp/leadfeeder_report_{int(time.time())}.xlsx"
336
- enriched_df.to_excel(path, index=False)
337
- ui.download(path)
338
- ui.notify('Download started!', type='positive')
339
-
340
- def refresh_dashboard(self):
341
- fig = build_dashboard_chart(self.df, self.dash_preset.value)
342
- self.dash_plot.update_figure(fig)
343
-
344
- def update_trend_options(self):
345
- if self.df.empty: return
346
- _, _, segment = TREND_PRESETS[self.trend_preset.value]
347
- if segment:
348
- opts = sorted(self.df[segment].dropna().unique().tolist())
349
- self.trend_filter.options = opts
350
- self.trend_filter.visible = True
351
- else:
352
- self.trend_filter.visible = False
353
- self.trend_filter.value = []
354
-
355
- def refresh_trends(self):
356
- fig = build_trend_chart(self.df, self.trend_preset.value, self.trend_grain.value, self.trend_filter.value)
357
- self.trend_plot.update_figure(fig)
358
-
359
- async def run_advanced_analytics(self):
360
- if self.df.empty: return
361
- ui.notify('Analyzing user flows...')
362
- sankey, heatmap, matrix = await run.io_bound(
363
- build_advanced_charts, self.df, self.adv_lead_id.value, self.adv_comp.value,
364
- self.date_start.value, self.date_end.value
365
- )
366
- self.sankey_plot.update_figure(sankey)
367
- self.heatmap_plot.update_figure(heatmap)
368
- self.matrix_plot.update_figure(matrix)
369
 
370
- async def run_debugger(self):
371
- if not self.debug_id.value: return
372
- res = await run.io_bound(
373
- requests.get,
374
- f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads/{self.debug_id.value}/visits",
375
- headers=HEADERS,
376
- params={"start_date": self.date_start.value, "end_date": self.date_end.value, "page[size]": 1, "include": "page_views"}
377
- )
378
- self.debug_output.set_content(f"```json\n{json.dumps(res.json(), indent=2)}\n```")
379
 
380
  # =====================================================
381
  # UI LAYOUT
382
  # =====================================================
383
- @ui.page('/')
384
- def main_page():
385
- app_logic = LeadfeederApp()
386
 
387
- # --- HEADER ---
388
- with ui.header().classes(replace='row items-center') as header:
389
- ui.icon('analytics', size='32px').classes('text-white q-mr-sm')
390
- ui.label('Leadfeeder Pro Analytics').classes('text-h6 text-white')
391
- ui.space()
392
- app_logic.pwd_input = ui.input('App Password', password=True).classes('bg-white rounded q-px-sm')
393
-
394
- # --- TABS ---
395
- with ui.tabs().classes('w-full') as tabs:
396
- data_tab = ui.tab('Data & Report')
397
- dash_tab = ui.tab('Dashboard')
398
- trend_tab = ui.tab('Trends')
399
- adv_tab = ui.tab('Advanced Analytics')
400
- debug_tab = ui.tab('Debugger')
401
-
402
- with ui.tab_panels(tabs, value=data_tab).classes('w-full q-pa-md'):
403
-
404
  # --- TAB 1: DATA ---
405
- with ui.tab_panel(data_tab):
406
- with ui.row().classes('w-full items-center gap-4'):
407
- app_logic.date_start = ui.input('Start Date', value=app_logic.start_date_val)
408
- app_logic.date_end = ui.input('End Date', value=app_logic.end_date_val)
409
- ui.button('Load Data', on_click=app_logic.load_data, icon='cloud_download')
410
- ui.button('Download Excel (Full)', on_click=app_logic.download_full_report, icon='file_download').classes('bg-green')
411
- app_logic.loading_spinner = ui.spinner(size='lg').props('color=primary').bind_visibility_from(app_logic.loading_spinner, 'visible')
412
- app_logic.loading_spinner.visible = False
413
-
414
- # KPIS
415
- with ui.row().classes('w-full gap-4 q-my-md'):
416
- with ui.card().classes('w-48'):
417
- ui.label('Total Companies').classes('text-caption')
418
- app_logic.kpi_total = ui.label('0').classes('text-h5')
419
- with ui.card().classes('w-48'):
420
- ui.label('Total Visits').classes('text-caption')
421
- app_logic.kpi_visits = ui.label('0').classes('text-h5')
422
- with ui.card().classes('w-48'):
423
- ui.label('Avg Quality').classes('text-caption')
424
- app_logic.kpi_quality = ui.label('0').classes('text-h5')
425
- with ui.card().classes('w-48'):
426
- ui.label('CRM Coverage').classes('text-caption')
427
- app_logic.kpi_crm = ui.label('0%').classes('text-h5')
428
-
429
- # PREVIEW GRID
430
- ui.label('Data Preview (Top 10 Enriched)').classes('text-h6 q-mt-md')
431
- app_logic.grid = ui.aggrid({
432
- 'columnDefs': [
433
- {'headerName': 'Company', 'field': 'company_name'},
434
- {'headerName': 'Industry', 'field': 'primary_industry'},
435
- {'headerName': 'Visits', 'field': 'total_visits'},
436
- {'headerName': 'Landing Page', 'field': 'landing_page_path'},
437
- {'headerName': 'Exit Page', 'field': 'exit_page_path'},
438
- ],
439
- 'rowData': []
440
- }).classes('h-96 w-full')
441
 
442
  # --- TAB 2: DASHBOARD ---
443
- with ui.tab_panel(dash_tab):
444
- with ui.row().classes('items-center'):
445
- app_logic.dash_preset = ui.select(list(DASHBOARD_PRESETS.keys()), label="Select Chart", value="Total Visits by Industry", on_change=app_logic.refresh_dashboard).classes('w-64')
446
- ui.button('Refresh', icon='refresh', on_click=app_logic.refresh_dashboard).props('flat')
 
 
 
447
 
448
- app_logic.dash_plot = ui.plotly({}).classes('w-full h-128')
 
449
 
450
  # --- TAB 3: TRENDS ---
451
- with ui.tab_panel(trend_tab):
452
- with ui.row().classes('items-start gap-4'):
453
- app_logic.trend_preset = ui.select(list(TREND_PRESETS.keys()), label="Metric", value="Total Visits Trend", on_change=lambda: [app_logic.update_trend_options(), app_logic.refresh_trends()]).classes('w-64')
454
- app_logic.trend_grain = ui.radio(['Daily', 'Weekly', 'Monthly'], value='Daily', on_change=app_logic.refresh_trends).props('inline')
455
- app_logic.trend_filter = ui.select([], multiple=True, label="Filter Segments", on_change=app_logic.refresh_trends).classes('w-96').props('use-chips')
456
- app_logic.trend_filter.visible = False
457
 
458
- app_logic.trend_plot = ui.plotly({}).classes('w-full h-128')
459
-
460
- # --- TAB 4: ADVANCED ---
461
- with ui.tab_panel(adv_tab):
462
- with ui.row().classes('items-center gap-4'):
463
- app_logic.adv_lead_id = ui.input('Search Lead ID')
464
- app_logic.adv_comp = ui.input('Search Company Name')
465
- ui.button('Analyze Flow', on_click=app_logic.run_advanced_analytics, icon='insights')
466
-
467
- with ui.row().classes('w-full'):
468
- with ui.column().classes('w-1/2'):
469
- app_logic.sankey_plot = ui.plotly({}).classes('w-full h-96')
470
- with ui.column().classes('w-1/2'):
471
- app_logic.heatmap_plot = ui.plotly({}).classes('w-full h-96')
472
 
473
- app_logic.matrix_plot = ui.plotly({}).classes('w-full h-96 q-mt-md')
 
 
 
 
 
474
 
475
- # --- TAB 5: DEBUGGER ---
476
- with ui.tab_panel(debug_tab):
477
- with ui.row():
478
- app_logic.debug_id = ui.input('Lead ID')
479
- ui.button('Inspect JSON', on_click=app_logic.run_debugger)
480
- app_logic.debug_output = ui.markdown('Raw JSON will appear here...')
 
481
 
482
- ui.run(title='Leadfeeder Analytics', port=8080, favicon='πŸš€')
 
 
3
  import requests
4
  import json
5
  import pandas as pd
6
+ import gradio as gr
7
  import plotly.express as px
8
  from datetime import date, timedelta
 
9
 
10
  # =====================================================
11
  # CONFIG / SECRETS
 
14
  APP_PASSWORD = os.getenv("APP_PASSWORD")
15
 
16
  if not API_TOKEN or not APP_PASSWORD:
17
+ raise RuntimeError("Missing required secrets: LEADFEEDER_API_TOKEN or APP_PASSWORD")
 
 
 
18
 
19
+ ACCOUNT_ID = "255333" # PREMIUM account
20
  BASE_URL = "https://api.leadfeeder.com"
21
+ PAGE_SIZE = 100
22
 
23
  HEADERS = {
24
  "Authorization": f"Token token={API_TOKEN}",
 
26
  }
27
 
28
  # =====================================================
29
+ # PRESETS
30
  # =====================================================
31
  DASHBOARD_PRESETS = {
32
  "Total Visits by Industry": ("primary_industry", "total_visits", "sum"),
 
48
  "Visits Trend by Assignee": ("total_visits", "sum", "assignee"),
49
  }
50
 
51
+ # =====================================================
52
+ # UTILS: RATE LIMIT HANDLER
53
+ # =====================================================
54
  def make_request(url, params=None):
55
  retries = 3
56
  while retries > 0:
57
  r = requests.get(url, headers=HEADERS, params=params, timeout=45)
58
  if r.status_code == 429:
59
+ print("⏳ Rate limit hit. Sleeping for 60 seconds...")
60
  time.sleep(61)
61
  retries -= 1
62
  continue
 
64
  return r.json()
65
  raise Exception("Max retries exceeded")
66
 
67
+ # =====================================================
68
+ # DATA FETCHING
69
+ # =====================================================
70
  def fetch_basic_leads(start_date, end_date):
71
+ """ Step 1: Fetch ALL company details (fast fetch). """
72
  page = 1
73
  rows = []
74
+
75
+ print(f"πŸš€ Fetching full company list for {start_date} to {end_date}...")
76
+
77
  while True:
78
  try:
79
  js = make_request(
80
  f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads",
81
+ params={
82
+ "start_date": start_date,
83
+ "end_date": end_date,
84
+ "page[number]": page,
85
+ "page[size]": PAGE_SIZE,
86
+ "include": "location"
87
+ }
88
  )
89
  data = js.get("data", [])
90
+ if not data:
91
+ break
92
+
93
  included = js.get("included", [])
94
  loc_map = {str(i["id"]): i["attributes"] for i in included if i["type"] == "locations"}
95
 
 
97
  a = lead["attributes"]
98
  loc_id = lead.get("relationships", {}).get("location", {}).get("data", {}).get("id")
99
  loc = loc_map.get(str(loc_id), {})
100
+
101
  rows.append({
102
  "lead_id": lead.get("id"),
103
  "company_name": a.get("name"),
104
  "website_url": a.get("website_url"),
105
+ "phone": a.get("phone"),
106
+ "business_id": a.get("business_id"),
107
  "primary_industry": a.get("industry"),
108
+ "all_industries": ", ".join([i.get("name") for i in a.get("industries", [])]) if a.get("industries") else None,
109
+ "first_visit_date": a.get("first_visit_date"),
110
  "last_visit_date": a.get("last_visit_date"),
111
  "total_visits": a.get("visits"),
112
  "lead_quality_score": a.get("quality"),
113
  "revenue": a.get("revenue"),
114
  "employee_count": a.get("employee_count"),
115
+ "employees_min": a.get("employees_range", {}).get("min") if a.get("employees_range") else None,
116
+ "employees_max": a.get("employees_range", {}).get("max") if a.get("employees_range") else None,
117
  "assignee": a.get("assignee"),
118
+ "emailed_to": a.get("emailed_to"),
119
+ "crm_lead_id": a.get("crm_lead_id"),
120
  "crm_organization_id": a.get("crm_organization_id"),
121
+ "tags": ", ".join(a.get("tags", [])) if a.get("tags") else None,
122
  "linkedin_url": a.get("linkedin_url"),
123
+ "twitter_handle": a.get("twitter_handle"),
124
+ "facebook_url": a.get("facebook_url"),
125
  "country": loc.get("country"),
126
+ "region": loc.get("region"),
127
  "city": loc.get("city"),
128
+ "leadfeeder_url": a.get("view_in_leadfeeder"),
129
  "landing_page_path": None,
130
  "exit_page_path": None
131
  })
132
+
133
+ print(f"βœ… Page {page} loaded. Rows: {len(rows)}")
134
  page += 1
135
+
136
  except Exception as e:
137
+ print(f"Error on page {page}: {e}")
138
  break
139
+
140
  df = pd.DataFrame(rows)
141
  if not df.empty:
142
  df["last_visit_date"] = pd.to_datetime(df["last_visit_date"], errors="coerce")
143
  return df
144
 
145
+ def enrich_leads_with_visits(df, start_date, end_date, max_rows=None, progress=gr.Progress()):
146
+ """
147
+ Step 2: Deep enrich with visit_route parsing for Exit Page.
148
+ """
149
+ if df.empty:
150
+ return df
151
+
152
  target_df = df.head(max_rows) if max_rows else df
153
+ total = len(target_df)
154
+
155
+ print(f"πŸ•΅οΈ Deep enriching {total} rows ({start_date} to {end_date})...")
156
 
157
  for index, row in target_df.iterrows():
158
+ if row.get("landing_page_path") and row.get("exit_page_path"):
159
+ continue
160
+
161
  lead_id = row["lead_id"]
162
  try:
163
  visit_data = make_request(
164
  f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads/{lead_id}/visits",
165
+ params={
166
+ "start_date": start_date,
167
+ "end_date": end_date,
168
+ "page[size]": 1,
169
+ "include": "page_views"
170
+ }
171
  )
172
+
173
  visits = visit_data.get("data", [])
174
  included = visit_data.get("included", [])
175
+ landing = None
176
+ exit_p = None
177
 
178
  if visits:
179
  visit = visits[0]
180
  v_attrs = visit.get("attributes", {})
181
+
182
+ # A. Direct Attributes
183
  landing = v_attrs.get("landing_page_path") or v_attrs.get("landing_page_url")
184
 
185
+ # B. Parse 'visit_route'
186
  visit_route = v_attrs.get("visit_route", [])
187
  if visit_route and isinstance(visit_route, list):
188
+ last_step = visit_route[-1]
189
+ if not exit_p: exit_p = last_step.get("page_path") or last_step.get("page_url")
190
+
191
+ if not landing:
192
+ first_step = visit_route[0]
193
+ landing = first_step.get("page_path") or first_step.get("page_url")
194
+
195
+ # C. Deep Link via 'included' Page Views
196
  if not landing or not exit_p:
197
  pv_map = {p["id"]: p["attributes"] for p in included if p["type"] == "page_views"}
198
  pv_ids = [r["id"] for r in visit.get("relationships", {}).get("page_views", {}).get("data", [])]
199
+
200
  if pv_ids:
201
+ if not landing:
202
+ first_pv = pv_map.get(pv_ids[0])
203
+ if first_pv: landing = first_pv.get("url") or first_pv.get("path")
204
+ if not exit_p:
205
+ last_pv = pv_map.get(pv_ids[-1])
206
+ if last_pv: exit_p = last_pv.get("url") or last_pv.get("path")
207
+
208
  df.at[index, "landing_page_path"] = landing
209
  df.at[index, "exit_page_path"] = exit_p
210
+ except Exception as e:
211
+ print(f"Failed to enrich lead {lead_id}: {e}")
212
+
213
+ if max_rows and index % 5 == 0:
214
+ progress(index / total, desc="Enriching...")
215
+
216
  return df
217
 
218
  # =====================================================
219
+ # DEBUGGER
220
  # =====================================================
221
+ def inspect_raw_json(lead_id, start_date, end_date):
222
+ if not lead_id: return "Please enter a Lead ID"
223
+ try:
224
+ url = f"{BASE_URL}/accounts/{ACCOUNT_ID}/leads/{lead_id}/visits"
225
+ params = {
226
+ "start_date": start_date,
227
+ "end_date": end_date,
228
+ "page[size]": 1,
229
+ "include": "page_views"
230
+ }
231
+ r = requests.get(url, headers=HEADERS, params=params)
232
+ return json.dumps(r.json(), indent=2)
233
+ except Exception as e:
234
+ return str(e)
235
 
236
+ # =====================================================
237
+ # DATA WRAPPERS
238
+ # =====================================================
239
+ def load_preview(start, end):
240
+ df = fetch_basic_leads(start, end)
241
+ if df.empty: return df, pd.DataFrame(), "⚠️ No data found."
242
+
243
+ df_preview = df.copy()
244
+ df_preview = enrich_leads_with_visits(df_preview, start, end, max_rows=10)
245
 
246
+ status = f"βœ… Loaded **{len(df):,} companies**. Preview shows top 10 rows enriched."
247
+ return df, df_preview.head(10), status
248
+
249
+ def download_full_excel(df, start, end):
250
+ if df is None or df.empty: return None
251
+ print("⏳ Starting full enrichment for Excel export...")
252
+ enriched_df = enrich_leads_with_visits(df.copy(), start, end)
253
+ path = "/tmp/leadfeeder_company_data_full.xlsx"
254
+ enriched_df.to_excel(path, index=False)
255
+ return path
256
+
257
+ # =====================================================
258
+ # CHART ENGINES
259
+ # =====================================================
260
+ def build_kpis(df):
261
+ if df is None or df.empty: return 0, 0, 0, 0, 0, 0
262
+ return (
263
+ len(df),
264
+ df["total_visits"].gt(0).sum(),
265
+ int(df["total_visits"].sum()),
266
+ round(df["lead_quality_score"].mean(), 2),
267
+ round(df["crm_organization_id"].notna().mean() * 100, 1),
268
+ round(df["linkedin_url"].notna().mean() * 100, 1),
269
+ )
270
+
271
+ def build_dashboard(df, preset, top_n):
272
+ if df is None or df.empty: return px.bar(title="No Data")
273
+ dim, metric, agg = DASHBOARD_PRESETS[preset]
274
  if agg == "count":
275
  grouped = df.groupby(dim, dropna=False).size().reset_index(name="value")
276
  else:
277
  grouped = df.groupby(dim, dropna=False)[metric].agg(agg).reset_index(name="value")
278
+ return px.bar(grouped.sort_values("value", ascending=False).head(top_n), x=dim, y="value", title=preset)
 
 
 
 
 
 
 
 
279
 
280
+ def get_trend_filter_options(df, preset):
281
+ if df is None or df.empty:
282
+ return gr.update(choices=[], value=None, visible=False)
283
  metric, agg, segment = TREND_PRESETS[preset]
284
+ if not segment:
285
+ return gr.update(choices=[], value=None, visible=False)
286
+ options = sorted(df[segment].dropna().unique().tolist())
287
+ top_5 = df[segment].value_counts().head(5).index.tolist()
288
+ return gr.update(choices=options, value=top_5, visible=True, label=f"Filter {segment}")
289
+
290
+ def build_trend(df, preset, grain, filter_values):
291
+ if df is None or df.empty: return px.line(title="No Data Loaded")
292
+ metric, agg, segment = TREND_PRESETS[preset]
293
+ df = df.dropna(subset=["last_visit_date"]).copy()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
294
 
295
+ if segment and filter_values:
296
+ df = df[df[segment].isin(filter_values)]
297
+ if df.empty: return px.line(title="No data for selected filters")
298
+
299
+ if grain == "Weekly":
300
+ df["period"] = df["last_visit_date"].dt.to_period("W").astype(str)
301
+ elif grain == "Monthly":
302
+ df["period"] = df["last_visit_date"].dt.to_period("M").astype(str)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
303
  else:
304
+ df["period"] = df["last_visit_date"].dt.date
 
 
 
 
305
 
306
+ val = metric
307
+ if agg == "count":
308
+ df["_v"] = df[metric].notna().astype(int)
309
+ val = "_v"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
310
 
311
+ if segment:
312
+ ts = df.groupby(["period", segment])[val].agg(agg).reset_index()
313
+ title = f"{preset} (Filtered)" if filter_values else preset
314
+ return px.line(ts, x="period", y=val, color=segment, template="plotly_white", title=title)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
315
 
316
+ ts = df.groupby("period")[val].agg(agg).reset_index()
317
+ return px.line(ts, x="period", y=val, markers=True, template="plotly_white", title=preset)
 
 
 
 
 
 
 
318
 
319
  # =====================================================
320
  # UI LAYOUT
321
  # =====================================================
322
+ with gr.Blocks(title="Leadfeeder Pro Analytics") as demo:
323
+ gr.Markdown("## πŸš€ Leadfeeder Analytics")
 
324
 
325
+ with gr.Row():
326
+ pwd = gr.Textbox(type="password", label="App Password")
327
+ gr.Button("Auth").click(lambda p: gr.Info("Success") if p==APP_PASSWORD else gr.Error("Invalid"), pwd, None)
328
+
329
+ df_state = gr.State()
330
+ status = gr.Markdown()
331
+
332
+ with gr.Tabs():
 
 
 
 
 
 
 
 
 
333
  # --- TAB 1: DATA ---
334
+ with gr.Tab("πŸ“‹ Data & Report"):
335
+ with gr.Row():
336
+ start = gr.Textbox(label="Start Date", value=(date.today()-timedelta(days=30)).isoformat())
337
+ end = gr.Textbox(label="End Date", value=date.today().isoformat())
338
+
339
+ with gr.Row():
340
+ btn_load = gr.Button("1. Load Data (Preview)", variant="primary")
341
+ btn_dl = gr.Button("2. Enrich & Download Full Excel")
342
+ file_dl = gr.File(label="Download Excel")
343
+
344
+ table = gr.Dataframe(label="Preview (Top 10 Enriched)", interactive=True)
345
+
346
+ btn_load.click(load_preview, [start, end], [df_state, table, status])
347
+ btn_dl.click(download_full_excel, [df_state, start, end], file_dl)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
348
 
349
  # --- TAB 2: DASHBOARD ---
350
+ with gr.Tab("πŸ“Š Dashboard"):
351
+ kpis = [gr.Number(label=l) for l in ["Total Companies", "Active Companies", "Total Visits", "Avg Quality", "CRM %", "LinkedIn %"]]
352
+ gr.Button("Refresh KPIs").click(build_kpis, df_state, kpis)
353
+
354
+ with gr.Row():
355
+ preset = gr.Dropdown(choices=list(DASHBOARD_PRESETS.keys()), label="Chart")
356
+ top_n = gr.Slider(5, 50, value=10, label="Top N")
357
 
358
+ chart = gr.Plot()
359
+ gr.Button("Build View").click(build_dashboard, [df_state, preset, top_n], chart)
360
 
361
  # --- TAB 3: TRENDS ---
362
+ with gr.Tab("πŸ“ˆ Trends"):
363
+ with gr.Row():
364
+ trend_view = gr.Dropdown(choices=list(TREND_PRESETS.keys()), label="Select Trend")
365
+ grain = gr.Radio(["Daily", "Weekly", "Monthly"], value="Daily", label="Granularity")
 
 
366
 
367
+ filter_dropdown = gr.Dropdown(multiselect=True, visible=False, label="Filter Segments")
368
+ plot = gr.Plot()
 
 
 
 
 
 
 
 
 
 
 
 
369
 
370
+ trend_view.change(get_trend_filter_options, [df_state, trend_view], filter_dropdown)
371
+ gr.Button("Show Trend", variant="primary").click(
372
+ build_trend,
373
+ [df_state, trend_view, grain, filter_dropdown],
374
+ plot
375
+ )
376
 
377
+ # --- TAB 4: DEBUGGER ---
378
+ with gr.Tab("πŸ› οΈ Debugger"):
379
+ gr.Markdown("Inspect raw API data for a specific lead.")
380
+ dbg_id = gr.Textbox(label="Lead ID")
381
+ dbg_btn = gr.Button("Inspect Raw JSON")
382
+ dbg_out = gr.Code(language="json")
383
+ dbg_btn.click(inspect_raw_json, [dbg_id, start, end], dbg_out)
384
 
385
+ if __name__ == "__main__":
386
+ demo.launch(server_name="0.0.0.0", server_port=7860)