Files changed (1) hide show
  1. app.py +411 -531
app.py CHANGED
@@ -4,27 +4,33 @@ import json
4
  import time
5
  import traceback
6
  from pathlib import Path
7
- from typing import Dict, Any, List, Tuple
8
 
9
  import pandas as pd
10
  import gradio as gr
11
- import papermill as pm
12
  import plotly.graph_objects as go
13
 
14
- # Optional LLM (HuggingFace Inference API)
 
 
 
 
15
  try:
16
  from huggingface_hub import InferenceClient
17
  except Exception:
18
  InferenceClient = None
19
 
20
  # =========================================================
21
- # CONFIG
22
  # =========================================================
23
 
24
  BASE_DIR = Path(__file__).resolve().parent
25
 
26
- NB1 = os.environ.get("NB1", "datacreation.ipynb").strip()
27
- NB2 = os.environ.get("NB2", "pythonanalysis.ipynb").strip()
 
 
 
28
 
29
  RUNS_DIR = BASE_DIR / "runs"
30
  ART_DIR = BASE_DIR / "artifacts"
@@ -32,20 +38,22 @@ PY_FIG_DIR = ART_DIR / "py" / "figures"
32
  PY_TAB_DIR = ART_DIR / "py" / "tables"
33
 
34
  PAPERMILL_TIMEOUT = int(os.environ.get("PAPERMILL_TIMEOUT", "1800"))
35
- MAX_PREVIEW_ROWS = int(os.environ.get("MAX_FILE_PREVIEW_ROWS", "50"))
36
  MAX_LOG_CHARS = int(os.environ.get("MAX_LOG_CHARS", "8000"))
37
 
 
38
  HF_API_KEY = os.environ.get("HF_API_KEY", "").strip()
39
  MODEL_NAME = os.environ.get("MODEL_NAME", "deepseek-ai/DeepSeek-R1").strip()
40
  HF_PROVIDER = os.environ.get("HF_PROVIDER", "novita").strip()
 
 
 
41
  N8N_WEBHOOK_URL = os.environ.get("N8N_WEBHOOK_URL", "").strip()
42
 
43
  LLM_ENABLED = bool(HF_API_KEY) and InferenceClient is not None
44
- llm_client = (
45
- InferenceClient(provider=HF_PROVIDER, api_key=HF_API_KEY)
46
- if LLM_ENABLED
47
- else None
48
- )
49
 
50
  # =========================================================
51
  # HELPERS
@@ -55,12 +63,29 @@ def ensure_dirs():
55
  for p in [RUNS_DIR, ART_DIR, PY_FIG_DIR, PY_TAB_DIR]:
56
  p.mkdir(parents=True, exist_ok=True)
57
 
58
- def stamp():
59
  return time.strftime("%Y%m%d-%H%M%S")
60
 
61
  def tail(text: str, n: int = MAX_LOG_CHARS) -> str:
62
  return (text or "")[-n:]
63
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
64
  def _ls(dir_path: Path, exts: Tuple[str, ...]) -> List[str]:
65
  if not dir_path.is_dir():
66
  return []
@@ -75,21 +100,39 @@ def _read_json(path: Path):
75
 
76
  def artifacts_index() -> Dict[str, Any]:
77
  return {
 
 
 
 
78
  "python": {
79
  "figures": _ls(PY_FIG_DIR, (".png", ".jpg", ".jpeg")),
80
  "tables": _ls(PY_TAB_DIR, (".csv", ".json")),
81
  },
82
  }
83
 
 
 
 
 
 
 
 
 
 
 
 
 
84
  # =========================================================
85
  # PIPELINE RUNNERS
86
  # =========================================================
87
 
88
  def run_notebook(nb_name: str) -> str:
89
  ensure_dirs()
 
 
90
  nb_in = BASE_DIR / nb_name
91
  if not nb_in.exists():
92
- return f"ERROR: {nb_name} not found."
93
  nb_out = RUNS_DIR / f"run_{stamp()}_{nb_name}"
94
  pm.execute_notebook(
95
  input_path=str(nb_in),
@@ -100,17 +143,15 @@ def run_notebook(nb_name: str) -> str:
100
  request_save_on_cell_execute=True,
101
  execution_timeout=PAPERMILL_TIMEOUT,
102
  )
103
- return f"Executed {nb_name}"
104
-
105
 
106
  def run_datacreation() -> str:
107
  try:
108
  log = run_notebook(NB1)
109
- csvs = [f.name for f in BASE_DIR.glob("*.csv")]
110
- return f"OK {log}\n\nCSVs now in /app:\n" + "\n".join(f" - {c}" for c in sorted(csvs))
111
  except Exception as e:
112
- return f"FAILED {e}\n\n{traceback.format_exc()[-2000:]}"
113
-
114
 
115
  def run_pythonanalysis() -> str:
116
  try:
@@ -118,494 +159,393 @@ def run_pythonanalysis() -> str:
118
  idx = artifacts_index()
119
  figs = idx["python"]["figures"]
120
  tabs = idx["python"]["tables"]
121
- return (
122
- f"OK {log}\n\n"
123
- f"Figures: {', '.join(figs) or '(none)'}\n"
124
- f"Tables: {', '.join(tabs) or '(none)'}"
125
- )
126
  except Exception as e:
127
- return f"FAILED {e}\n\n{traceback.format_exc()[-2000:]}"
128
-
129
 
130
  def run_full_pipeline() -> str:
131
- logs = []
132
- logs.append("=" * 50)
133
- logs.append("STEP 1/2: Data Creation (web scraping + synthetic data)")
134
- logs.append("=" * 50)
135
- logs.append(run_datacreation())
136
- logs.append("")
137
- logs.append("=" * 50)
138
- logs.append("STEP 2/2: Python Analysis (sentiment, ARIMA, dashboard)")
139
- logs.append("=" * 50)
140
- logs.append(run_pythonanalysis())
141
- return "\n".join(logs)
142
-
143
 
144
  # =========================================================
145
- # GALLERY LOADERS
146
  # =========================================================
147
 
148
- def _load_all_figures() -> List[Tuple[str, str]]:
149
- """Return list of (filepath, caption) for Gallery."""
150
- items = []
151
- for p in sorted(PY_FIG_DIR.glob("*.png")):
152
- items.append((str(p), p.stem.replace('_', ' ').title()))
153
- return items
154
-
155
-
156
- def _load_table_safe(path: Path) -> pd.DataFrame:
157
  try:
158
- if path.suffix == ".json":
159
  obj = _read_json(path)
160
- if isinstance(obj, dict):
161
- return pd.DataFrame([obj])
162
- return pd.DataFrame(obj)
163
  return _read_csv(path)
164
  except Exception as e:
165
  return pd.DataFrame([{"error": str(e)}])
166
 
167
-
168
  def refresh_gallery():
169
- """Called when user clicks Refresh on Gallery tab."""
170
- figures = _load_all_figures()
171
  idx = artifacts_index()
172
-
173
  table_choices = list(idx["python"]["tables"])
174
 
 
 
 
 
 
175
  default_df = pd.DataFrame()
176
  if table_choices:
177
- default_df = _load_table_safe(PY_TAB_DIR / table_choices[0])
178
-
179
- return (
180
- figures if figures else [],
181
- gr.update(choices=table_choices, value=table_choices[0] if table_choices else None),
182
- default_df,
183
- )
184
 
 
185
 
186
  def on_table_select(choice: str):
187
  if not choice:
188
  return pd.DataFrame([{"hint": "Select a table above."}])
189
- path = PY_TAB_DIR / choice
190
  if not path.exists():
191
  return pd.DataFrame([{"error": f"File not found: {choice}"}])
192
- return _load_table_safe(path)
193
-
194
 
195
  # =========================================================
196
- # KPI LOADER
197
  # =========================================================
198
 
199
  def load_kpis() -> Dict[str, Any]:
200
- for candidate in [PY_TAB_DIR / "kpis.json", PY_FIG_DIR / "kpis.json"]:
201
- if candidate.exists():
202
- try:
203
- return _read_json(candidate)
204
- except Exception:
205
- pass
206
- return {}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
207
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
208
 
209
  # =========================================================
210
- # AI DASHBOARD -- LLM picks what to display
211
  # =========================================================
212
 
213
- DASHBOARD_SYSTEM = """You are an AI dashboard assistant for a book-sales analytics app.
214
- The user asks questions or requests about their data. You have access to pre-computed
215
- artifacts from a Python analysis pipeline.
216
 
217
- AVAILABLE ARTIFACTS (only reference ones that exist):
218
  {artifacts_json}
219
 
220
- KPI SUMMARY: {kpis_json}
221
-
222
- YOUR JOB:
223
- 1. Answer the user's question conversationally using the KPIs and your knowledge of the artifacts.
224
- 2. At the END of your response, output a JSON block (fenced with ```json ... ```) that tells
225
- the dashboard which artifact to display. The JSON must have this shape:
226
- {{"show": "figure"|"table"|"none", "scope": "python", "filename": "..."}}
227
-
228
- - Use "show": "figure" to display a chart image.
229
- - Use "show": "table" to display a CSV/JSON table.
230
- - Use "show": "none" if no artifact is relevant.
231
-
232
- RULES:
233
- - If the user asks about sales trends or forecasting by title, show sales_trends or arima figures.
234
- - If the user asks about sentiment, show sentiment figure or sentiment_counts table.
235
- - If the user asks about forecast accuracy or ARIMA, show arima figures.
236
- - If the user asks about top sellers, show top_titles_by_units_sold.csv.
237
- - If the user asks a general data question, pick the most relevant artifact.
238
- - Keep your answer concise (2-4 sentences), then the JSON block.
239
  """
240
 
241
  JSON_BLOCK_RE = re.compile(r"```json\s*(\{.*?\})\s*```", re.DOTALL)
242
  FALLBACK_JSON_RE = re.compile(r"\{[^{}]*\"show\"[^{}]*\}", re.DOTALL)
243
 
 
 
 
 
 
 
 
 
 
244
 
245
- def _parse_display_directive(text: str) -> Dict[str, str]:
246
- m = JSON_BLOCK_RE.search(text)
247
- if m:
248
- try:
249
- return json.loads(m.group(1))
250
- except json.JSONDecodeError:
251
- pass
252
- m = FALLBACK_JSON_RE.search(text)
253
- if m:
254
- try:
255
- return json.loads(m.group(0))
256
- except json.JSONDecodeError:
257
- pass
258
- return {"show": "none"}
259
-
260
-
261
- def _clean_response(text: str) -> str:
262
- """Strip the JSON directive block from the displayed response."""
263
  return JSON_BLOCK_RE.sub("", text).strip()
264
 
265
-
266
- def _n8n_call(msg: str) -> Tuple[str, Dict]:
267
- """Call the student's n8n webhook and return (reply, directive)."""
268
- import requests as req
269
  try:
270
- resp = req.post(N8N_WEBHOOK_URL, json={"question": msg}, timeout=20)
 
271
  data = resp.json()
272
- answer = data.get("answer", "No response from n8n workflow.")
273
  chart = data.get("chart", "none")
274
- if chart and chart != "none":
275
- return answer, {"show": "figure", "chart": chart}
276
- return answer, {"show": "none"}
277
  except Exception as e:
278
- return f"n8n error: {e}. Falling back to keyword matching.", None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
279
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
280
 
281
  def ai_chat(user_msg: str, history: list):
282
- """Chat function for the AI Dashboard tab."""
283
  if not user_msg or not user_msg.strip():
284
  return history, "", None, None
285
 
286
  idx = artifacts_index()
287
  kpis = load_kpis()
288
 
289
- # Priority: n8n webhook > HF LLM > keyword fallback
290
  if N8N_WEBHOOK_URL:
291
- reply, directive = _n8n_call(user_msg)
292
  if directive is None:
293
- reply_fb, directive = _keyword_fallback(user_msg, idx, kpis)
294
- reply += "\n\n" + reply_fb
295
- elif not LLM_ENABLED:
296
- reply, directive = _keyword_fallback(user_msg, idx, kpis)
297
- else:
298
  system = DASHBOARD_SYSTEM.format(
299
  artifacts_json=json.dumps(idx, indent=2),
300
- kpis_json=json.dumps(kpis, indent=2) if kpis else "(no KPIs yet, run the pipeline first)",
301
  )
302
  msgs = [{"role": "system", "content": system}]
303
  for entry in (history or [])[-6:]:
304
  msgs.append(entry)
305
  msgs.append({"role": "user", "content": user_msg})
306
-
307
  try:
308
  r = llm_client.chat_completion(
309
  model=MODEL_NAME,
310
  messages=msgs,
311
- temperature=0.3,
312
- max_tokens=600,
313
  stream=False,
314
  )
315
- raw = (
316
- r["choices"][0]["message"]["content"]
317
- if isinstance(r, dict)
318
- else r.choices[0].message.content
319
- )
320
- directive = _parse_display_directive(raw)
321
- reply = _clean_response(raw)
322
  except Exception as e:
323
- reply = f"LLM error: {e}. Falling back to keyword matching."
324
- reply_fb, directive = _keyword_fallback(user_msg, idx, kpis)
325
- reply += "\n\n" + reply_fb
326
-
327
- # Resolve artifacts — build interactive Plotly charts when possible
328
- chart_out = None
329
- tab_out = None
330
- show = directive.get("show", "none")
331
- fname = directive.get("filename", "")
332
- chart_name = directive.get("chart", "")
333
-
334
- # Interactive chart builders keyed by name
335
- chart_builders = {
336
- "sales": build_sales_chart,
337
- "sentiment": build_sentiment_chart,
338
- "top_sellers": build_top_sellers_chart,
339
- }
340
 
341
- if chart_name and chart_name in chart_builders:
342
- chart_out = chart_builders[chart_name]()
343
- elif show == "figure" and fname:
344
- # Fallback: try to match filename to a chart builder
345
- if "sales_trend" in fname:
346
- chart_out = build_sales_chart()
347
- elif "sentiment" in fname:
348
- chart_out = build_sentiment_chart()
349
- elif "arima" in fname or "forecast" in fname:
350
- chart_out = build_sales_chart() # closest interactive equivalent
351
- else:
352
- chart_out = _empty_chart(f"No interactive chart for {fname}")
353
-
354
- if show == "table" and fname:
355
- fp = PY_TAB_DIR / fname
356
- if fp.exists():
357
- tab_out = _load_table_safe(fp)
358
- else:
359
- reply += f"\n\n*(Could not find table: {fname})*"
360
 
361
  new_history = (history or []) + [
362
  {"role": "user", "content": user_msg},
363
  {"role": "assistant", "content": reply},
364
  ]
365
-
366
- return new_history, "", chart_out, tab_out
367
-
368
-
369
- def _keyword_fallback(msg: str, idx: Dict, kpis: Dict) -> Tuple[str, Dict]:
370
- """Simple keyword matcher when LLM is unavailable."""
371
- msg_lower = msg.lower()
372
-
373
- if not idx["python"]["figures"] and not idx["python"]["tables"]:
374
- return (
375
- "No artifacts found yet. Please run the pipeline first (Tab 1), "
376
- "then come back here to explore the results.",
377
- {"show": "none"},
378
- )
379
-
380
- kpi_text = ""
381
- if kpis:
382
- total = kpis.get("total_units_sold", 0)
383
- kpi_text = (
384
- f"Quick summary: **{kpis.get('n_titles', '?')}** book titles across "
385
- f"**{kpis.get('n_months', '?')}** months, with **{total:,.0f}** total units sold."
386
- )
387
-
388
- if any(w in msg_lower for w in ["trend", "sales trend", "monthly sale"]):
389
- return (
390
- f"Here are the sales trends. {kpi_text}",
391
- {"show": "figure", "chart": "sales"},
392
- )
393
-
394
- if any(w in msg_lower for w in ["sentiment", "review", "positive", "negative"]):
395
- return (
396
- f"Here is the sentiment distribution across sampled book titles. {kpi_text}",
397
- {"show": "figure", "chart": "sentiment"},
398
- )
399
-
400
- if any(w in msg_lower for w in ["arima", "forecast", "predict"]):
401
- return (
402
- f"Here are the sales trends and forecasts. {kpi_text}",
403
- {"show": "figure", "chart": "sales"},
404
- )
405
-
406
- if any(w in msg_lower for w in ["top", "best sell", "popular", "rank"]):
407
- return (
408
- f"Here are the top-selling titles by units sold. {kpi_text}",
409
- {"show": "table", "scope": "python", "filename": "top_titles_by_units_sold.csv"},
410
- )
411
-
412
- if any(w in msg_lower for w in ["price", "pricing", "decision"]):
413
- return (
414
- f"Here are the pricing decisions. {kpi_text}",
415
- {"show": "table", "scope": "python", "filename": "pricing_decisions.csv"},
416
- )
417
-
418
- if any(w in msg_lower for w in ["dashboard", "overview", "summary", "kpi"]):
419
- return (
420
- f"Dashboard overview: {kpi_text}\n\nAsk me about sales trends, sentiment, forecasts, "
421
- "pricing, or top sellers to see specific visualizations.",
422
- {"show": "table", "scope": "python", "filename": "df_dashboard.csv"},
423
- )
424
-
425
- # Default
426
- return (
427
- f"I can show you various analyses. {kpi_text}\n\n"
428
- "Try asking about: **sales trends**, **sentiment**, **ARIMA forecasts**, "
429
- "**pricing decisions**, **top sellers**, or **dashboard overview**.",
430
- {"show": "none"},
431
- )
432
-
433
-
434
- # =========================================================
435
- # KPI CARDS (BubbleBusters style)
436
- # =========================================================
437
-
438
- def render_kpi_cards() -> str:
439
- kpis = load_kpis()
440
- if not kpis:
441
- return (
442
- '<div style="background:rgba(255,255,255,.65);backdrop-filter:blur(16px);'
443
- 'border-radius:20px;padding:28px;text-align:center;'
444
- 'border:1.5px solid rgba(255,255,255,.7);'
445
- 'box-shadow:0 8px 32px rgba(124,92,191,.08);">'
446
- '<div style="font-size:36px;margin-bottom:10px;">📊</div>'
447
- '<div style="color:#a48de8;font-size:14px;'
448
- 'font-weight:800;margin-bottom:6px;">No data yet</div>'
449
- '<div style="color:#9d8fc4;font-size:12px;">'
450
- 'Run the pipeline to populate these cards.</div>'
451
- '</div>'
452
- )
453
-
454
- def card(icon, label, value, colour):
455
- return f"""
456
- <div style="background:rgba(255,255,255,.72);backdrop-filter:blur(16px);
457
- border-radius:20px;padding:18px 14px 16px;text-align:center;
458
- border:1.5px solid rgba(255,255,255,.8);
459
- box-shadow:0 4px 16px rgba(124,92,191,.08);
460
- border-top:3px solid {colour};">
461
- <div style="font-size:26px;margin-bottom:7px;line-height:1;">{icon}</div>
462
- <div style="color:#9d8fc4;font-size:9.5px;text-transform:uppercase;
463
- letter-spacing:1.8px;margin-bottom:7px;font-weight:800;">{label}</div>
464
- <div style="color:#2d1f4e;font-size:16px;font-weight:800;">{value}</div>
465
- </div>"""
466
-
467
- kpi_config = [
468
- ("n_titles", "📚", "Book Titles", "#a48de8"),
469
- ("n_months", "📅", "Time Periods", "#7aa6f8"),
470
- ("total_units_sold", "📦", "Units Sold", "#6ee7c7"),
471
- ("total_revenue", "💰", "Revenue", "#3dcba8"),
472
- ]
473
-
474
- html = (
475
- '<div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(140px,1fr));'
476
- 'gap:12px;margin-bottom:24px;">'
477
- )
478
- for key, icon, label, colour in kpi_config:
479
- val = kpis.get(key)
480
- if val is None:
481
- continue
482
- if isinstance(val, (int, float)) and val > 100:
483
- val = f"{val:,.0f}"
484
- html += card(icon, label, str(val), colour)
485
- # Extra KPIs not in config
486
- known = {k for k, *_ in kpi_config}
487
- for key, val in kpis.items():
488
- if key not in known:
489
- label = key.replace("_", " ").title()
490
- if isinstance(val, (int, float)) and val > 100:
491
- val = f"{val:,.0f}"
492
- html += card("📈", label, str(val), "#8fa8f8")
493
- html += "</div>"
494
- return html
495
-
496
-
497
- # =========================================================
498
- # INTERACTIVE PLOTLY CHARTS (BubbleBusters style)
499
- # =========================================================
500
-
501
- CHART_PALETTE = ["#7c5cbf", "#2ec4a0", "#e8537a", "#e8a230", "#5e8fef",
502
- "#c45ea8", "#3dbacc", "#a0522d", "#6aaa3a", "#d46060"]
503
-
504
- def _styled_layout(**kwargs) -> dict:
505
- defaults = dict(
506
- template="plotly_white",
507
- paper_bgcolor="rgba(255,255,255,0.95)",
508
- plot_bgcolor="rgba(255,255,255,0.98)",
509
- font=dict(family="system-ui, sans-serif", color="#2d1f4e", size=12),
510
- margin=dict(l=60, r=20, t=70, b=70),
511
- legend=dict(
512
- orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1,
513
- bgcolor="rgba(255,255,255,0.92)",
514
- bordercolor="rgba(124,92,191,0.35)", borderwidth=1,
515
- ),
516
- title=dict(font=dict(size=15, color="#4b2d8a")),
517
- )
518
- defaults.update(kwargs)
519
- return defaults
520
-
521
-
522
- def _empty_chart(title: str) -> go.Figure:
523
- fig = go.Figure()
524
- fig.update_layout(
525
- title=title, height=420, template="plotly_white",
526
- paper_bgcolor="rgba(255,255,255,0.95)",
527
- annotations=[dict(text="Run the pipeline to generate data",
528
- x=0.5, y=0.5, xref="paper", yref="paper", showarrow=False,
529
- font=dict(size=14, color="rgba(124,92,191,0.5)"))],
530
- )
531
- return fig
532
-
533
-
534
- def build_sales_chart() -> go.Figure:
535
- path = PY_TAB_DIR / "df_dashboard.csv"
536
- if not path.exists():
537
- return _empty_chart("Sales Trends — run the pipeline first")
538
- df = pd.read_csv(path)
539
- date_col = next((c for c in df.columns if "month" in c.lower() or "date" in c.lower()), None)
540
- val_cols = [c for c in df.columns if c != date_col and df[c].dtype in ("float64", "int64")]
541
- if not date_col or not val_cols:
542
- return _empty_chart("Could not auto-detect columns in df_dashboard.csv")
543
- df[date_col] = pd.to_datetime(df[date_col], errors="coerce")
544
- fig = go.Figure()
545
- for i, col in enumerate(val_cols):
546
- fig.add_trace(go.Scatter(
547
- x=df[date_col], y=df[col], name=col.replace("_", " ").title(),
548
- mode="lines+markers", line=dict(color=CHART_PALETTE[i % len(CHART_PALETTE)], width=2),
549
- marker=dict(size=4),
550
- hovertemplate=f"<b>{col.replace('_',' ').title()}</b><br>%{{x|%b %Y}}: %{{y:,.0f}}<extra></extra>",
551
- ))
552
- fig.update_layout(**_styled_layout(height=450, hovermode="x unified",
553
- title=dict(text="Monthly Overview")))
554
- fig.update_xaxes(gridcolor="rgba(124,92,191,0.15)", showgrid=True)
555
- fig.update_yaxes(gridcolor="rgba(124,92,191,0.15)", showgrid=True)
556
- return fig
557
-
558
-
559
- def build_sentiment_chart() -> go.Figure:
560
- path = PY_TAB_DIR / "sentiment_counts_sampled.csv"
561
- if not path.exists():
562
- return _empty_chart("Sentiment Distribution — run the pipeline first")
563
- df = pd.read_csv(path)
564
- title_col = df.columns[0]
565
- sent_cols = [c for c in ["negative", "neutral", "positive"] if c in df.columns]
566
- if not sent_cols:
567
- return _empty_chart("No sentiment columns found in CSV")
568
- colors = {"negative": "#e8537a", "neutral": "#5e8fef", "positive": "#2ec4a0"}
569
- fig = go.Figure()
570
- for col in sent_cols:
571
- fig.add_trace(go.Bar(
572
- name=col.title(), y=df[title_col], x=df[col],
573
- orientation="h", marker_color=colors.get(col, "#888"),
574
- hovertemplate=f"<b>{col.title()}</b>: %{{x}}<extra></extra>",
575
- ))
576
- fig.update_layout(**_styled_layout(
577
- height=max(400, len(df) * 28), barmode="stack",
578
- title=dict(text="Sentiment Distribution by Book"),
579
- ))
580
- fig.update_xaxes(title="Number of Reviews")
581
- fig.update_yaxes(autorange="reversed")
582
- return fig
583
-
584
-
585
- def build_top_sellers_chart() -> go.Figure:
586
- path = PY_TAB_DIR / "top_titles_by_units_sold.csv"
587
- if not path.exists():
588
- return _empty_chart("Top Sellers — run the pipeline first")
589
- df = pd.read_csv(path).head(15)
590
- title_col = next((c for c in df.columns if "title" in c.lower()), df.columns[0])
591
- val_col = next((c for c in df.columns if "unit" in c.lower() or "sold" in c.lower()), df.columns[-1])
592
- fig = go.Figure(go.Bar(
593
- y=df[title_col], x=df[val_col], orientation="h",
594
- marker=dict(color=df[val_col], colorscale=[[0, "#c5b4f0"], [1, "#7c5cbf"]]),
595
- hovertemplate="<b>%{y}</b><br>Units: %{x:,.0f}<extra></extra>",
596
- ))
597
- fig.update_layout(**_styled_layout(
598
- height=max(400, len(df) * 30),
599
- title=dict(text="Top Selling Titles"), showlegend=False,
600
- ))
601
- fig.update_yaxes(autorange="reversed")
602
- fig.update_xaxes(title="Total Units Sold")
603
- return fig
604
-
605
-
606
- def refresh_dashboard():
607
- return render_kpi_cards(), build_sales_chart(), build_sentiment_chart(), build_top_sellers_chart()
608
-
609
 
610
  # =========================================================
611
  # UI
@@ -617,142 +557,82 @@ def load_css() -> str:
617
  css_path = BASE_DIR / "style.css"
618
  return css_path.read_text(encoding="utf-8") if css_path.exists() else ""
619
 
620
-
621
- with gr.Blocks(title="AIBDM 2026 Workshop App") as demo:
622
-
623
  gr.Markdown(
624
- "# SE21 App Template\n"
625
- "*This is an app template for SE21 students*",
626
  elem_id="escp_title",
627
  )
628
 
629
- # ===========================================================
630
- # TAB 1 -- Pipeline Runner
631
- # ===========================================================
632
  with gr.Tab("Pipeline Runner"):
633
- gr.Markdown()
634
-
635
  with gr.Row():
636
- with gr.Column(scale=1):
637
- btn_nb1 = gr.Button("Step 1: Data Creation", variant="secondary")
638
- with gr.Column(scale=1):
639
- btn_nb2 = gr.Button("Step 2: Python Analysis", variant="secondary")
640
-
641
- with gr.Row():
642
- btn_all = gr.Button("Run Full Pipeline (Both Steps)", variant="primary")
643
-
644
- run_log = gr.Textbox(
645
- label="Execution Log",
646
- lines=18,
647
- max_lines=30,
648
- interactive=False,
649
- )
650
-
651
  btn_nb1.click(run_datacreation, outputs=[run_log])
652
  btn_nb2.click(run_pythonanalysis, outputs=[run_log])
653
  btn_all.click(run_full_pipeline, outputs=[run_log])
654
 
655
- # ===========================================================
656
- # TAB 2 -- Dashboard (KPIs + Interactive Charts + Gallery)
657
- # ===========================================================
658
  with gr.Tab("Dashboard"):
659
  kpi_html = gr.HTML(value=render_kpi_cards)
660
-
661
  refresh_btn = gr.Button("Refresh Dashboard", variant="primary")
662
 
663
- gr.Markdown("#### Interactive Charts")
664
- chart_sales = gr.Plot(label="Monthly Overview")
665
- chart_sentiment = gr.Plot(label="Sentiment Distribution")
666
- chart_top = gr.Plot(label="Top Sellers")
667
-
668
- gr.Markdown("#### Static Figures (from notebooks)")
669
- gallery = gr.Gallery(
670
- label="Generated Figures",
671
- columns=2,
672
- height=480,
673
- object_fit="contain",
674
- )
675
 
676
- gr.Markdown("#### Data Tables")
677
- table_dropdown = gr.Dropdown(
678
- label="Select a table to view",
679
- choices=[],
680
- interactive=True,
681
- )
682
- table_display = gr.Dataframe(
683
- label="Table Preview",
684
- interactive=False,
685
- )
686
 
687
- def _on_refresh():
688
- kpi, c1, c2, c3 = refresh_dashboard()
689
- figs, dd, df = refresh_gallery()
690
- return kpi, c1, c2, c3, figs, dd, df
691
 
692
  refresh_btn.click(
693
- _on_refresh,
694
- outputs=[kpi_html, chart_sales, chart_sentiment, chart_top,
695
- gallery, table_dropdown, table_display],
696
- )
697
- table_dropdown.change(
698
- on_table_select,
699
- inputs=[table_dropdown],
700
- outputs=[table_display],
701
  )
 
702
 
703
- # ===========================================================
704
- # TAB 3 -- AI Dashboard
705
- # ===========================================================
706
  with gr.Tab('"AI" Dashboard'):
707
- _ai_status = (
708
- "Connected to your **n8n workflow**." if N8N_WEBHOOK_URL
709
- else "**LLM active.**" if LLM_ENABLED
710
- else "Using **keyword matching**. Upgrade options: "
711
- "set `N8N_WEBHOOK_URL` to connect your n8n workflow, "
712
- "or set `HF_API_KEY` for direct LLM access."
713
  )
714
  gr.Markdown(
715
- "### Ask questions, get interactive visualisations\n\n"
716
- f"Type a question and the system will pick the right interactive chart or table. {_ai_status}"
 
717
  )
718
-
719
  with gr.Row(equal_height=True):
720
  with gr.Column(scale=1):
721
- chatbot = gr.Chatbot(
722
- label="Conversation",
723
- height=380,
724
- )
725
- user_input = gr.Textbox(
726
- label="Ask about your data",
727
- placeholder="e.g. Show me sales trends / What are the top sellers? / Sentiment analysis",
728
- lines=1,
729
- )
730
  gr.Examples(
731
  examples=[
732
- "Show me the sales trends",
733
- "What does the sentiment look like?",
734
- "Which titles sell the most?",
735
- "Show the ARIMA forecasts",
736
- "What are the pricing decisions?",
737
- "Give me a dashboard overview",
738
  ],
739
  inputs=user_input,
740
  )
741
-
742
  with gr.Column(scale=1):
743
- ai_figure = gr.Plot(
744
- label="Interactive Chart",
745
- )
746
- ai_table = gr.Dataframe(
747
- label="Data Table",
748
- interactive=False,
749
- )
750
-
751
- user_input.submit(
752
- ai_chat,
753
- inputs=[user_input, chatbot],
754
- outputs=[chatbot, user_input, ai_figure, ai_table],
755
- )
756
 
 
757
 
758
- demo.launch(css=load_css(), allowed_paths=[str(BASE_DIR)])
 
 
4
  import time
5
  import traceback
6
  from pathlib import Path
7
+ from typing import Dict, Any, List, Tuple, Optional
8
 
9
  import pandas as pd
10
  import gradio as gr
 
11
  import plotly.graph_objects as go
12
 
13
+ try:
14
+ import papermill as pm
15
+ except Exception:
16
+ pm = None
17
+
18
  try:
19
  from huggingface_hub import InferenceClient
20
  except Exception:
21
  InferenceClient = None
22
 
23
  # =========================================================
24
+ # CONFIG — ITALY HOSPITALITY MARKET INSIGHT ASSISTANT
25
  # =========================================================
26
 
27
  BASE_DIR = Path(__file__).resolve().parent
28
 
29
+ NB1 = os.environ.get("NB1", "1_Data_Creation_Italy_Hospitality.ipynb").strip()
30
+ NB2 = os.environ.get("NB2", "2a_Python_Analysis_Italy_Hospitality.ipynb").strip()
31
+
32
+ CLEANED_CSV = os.environ.get("CLEANED_CSV", "italy_hospitality_market_cleaned.csv").strip()
33
+ ENRICHED_CSV = os.environ.get("ENRICHED_CSV", "italy_hospitality_market_enriched_synthetic.csv").strip()
34
 
35
  RUNS_DIR = BASE_DIR / "runs"
36
  ART_DIR = BASE_DIR / "artifacts"
 
38
  PY_TAB_DIR = ART_DIR / "py" / "tables"
39
 
40
  PAPERMILL_TIMEOUT = int(os.environ.get("PAPERMILL_TIMEOUT", "1800"))
41
+ MAX_PREVIEW_ROWS = int(os.environ.get("MAX_FILE_PREVIEW_ROWS", "80"))
42
  MAX_LOG_CHARS = int(os.environ.get("MAX_LOG_CHARS", "8000"))
43
 
44
+ # Hugging Face Inference API
45
  HF_API_KEY = os.environ.get("HF_API_KEY", "").strip()
46
  MODEL_NAME = os.environ.get("MODEL_NAME", "deepseek-ai/DeepSeek-R1").strip()
47
  HF_PROVIDER = os.environ.get("HF_PROVIDER", "novita").strip()
48
+
49
+ # Optional n8n automation webhook. Expected JSON response:
50
+ # {"answer": "...", "chart": "risk|investment|city|region|overview|none"}
51
  N8N_WEBHOOK_URL = os.environ.get("N8N_WEBHOOK_URL", "").strip()
52
 
53
  LLM_ENABLED = bool(HF_API_KEY) and InferenceClient is not None
54
+ llm_client = InferenceClient(provider=HF_PROVIDER, api_key=HF_API_KEY) if LLM_ENABLED else None
55
+
56
+ CHART_PALETTE = ["#7c5cbf", "#2ec4a0", "#e8537a", "#e8a230", "#5e8fef", "#c45ea8"]
 
 
57
 
58
  # =========================================================
59
  # HELPERS
 
63
  for p in [RUNS_DIR, ART_DIR, PY_FIG_DIR, PY_TAB_DIR]:
64
  p.mkdir(parents=True, exist_ok=True)
65
 
66
+ def stamp() -> str:
67
  return time.strftime("%Y%m%d-%H%M%S")
68
 
69
  def tail(text: str, n: int = MAX_LOG_CHARS) -> str:
70
  return (text or "")[-n:]
71
 
72
+ def csv_path(prefer_enriched: bool = True) -> Path:
73
+ enriched = BASE_DIR / ENRICHED_CSV
74
+ cleaned = BASE_DIR / CLEANED_CSV
75
+ if prefer_enriched and enriched.exists():
76
+ return enriched
77
+ if cleaned.exists():
78
+ return cleaned
79
+ return enriched
80
+
81
+ def read_market_data(prefer_enriched: bool = True) -> pd.DataFrame:
82
+ path = csv_path(prefer_enriched)
83
+ if not path.exists():
84
+ return pd.DataFrame(columns=["section", "location", "entity", "metric_name", "value", "unit", "period", "source_page", "note"])
85
+ df = pd.read_csv(path)
86
+ df["value"] = pd.to_numeric(df.get("value"), errors="coerce")
87
+ return df
88
+
89
  def _ls(dir_path: Path, exts: Tuple[str, ...]) -> List[str]:
90
  if not dir_path.is_dir():
91
  return []
 
100
 
101
  def artifacts_index() -> Dict[str, Any]:
102
  return {
103
+ "core_files": {
104
+ "cleaned_dataset": CLEANED_CSV if (BASE_DIR / CLEANED_CSV).exists() else None,
105
+ "enriched_synthetic_dataset": ENRICHED_CSV if (BASE_DIR / ENRICHED_CSV).exists() else None,
106
+ },
107
  "python": {
108
  "figures": _ls(PY_FIG_DIR, (".png", ".jpg", ".jpeg")),
109
  "tables": _ls(PY_TAB_DIR, (".csv", ".json")),
110
  },
111
  }
112
 
113
+ def pivot_metrics(df: pd.DataFrame, section: Optional[str] = None, entity: Optional[str] = None) -> pd.DataFrame:
114
+ d = df.copy()
115
+ if section:
116
+ d = d[d["section"].eq(section)]
117
+ if entity:
118
+ d = d[d["entity"].eq(entity)]
119
+ if d.empty:
120
+ return pd.DataFrame()
121
+ wide = d.pivot_table(index=["location", "entity"], columns="metric_name", values="value", aggfunc="first").reset_index()
122
+ wide.columns.name = None
123
+ return wide
124
+
125
  # =========================================================
126
  # PIPELINE RUNNERS
127
  # =========================================================
128
 
129
  def run_notebook(nb_name: str) -> str:
130
  ensure_dirs()
131
+ if pm is None:
132
+ return "ERROR: papermill is not installed. Add it to requirements.txt."
133
  nb_in = BASE_DIR / nb_name
134
  if not nb_in.exists():
135
+ return f"ERROR: {nb_name} not found in {BASE_DIR}."
136
  nb_out = RUNS_DIR / f"run_{stamp()}_{nb_name}"
137
  pm.execute_notebook(
138
  input_path=str(nb_in),
 
143
  request_save_on_cell_execute=True,
144
  execution_timeout=PAPERMILL_TIMEOUT,
145
  )
146
+ return f"Executed {nb_name}. Output saved to {nb_out.name}"
 
147
 
148
  def run_datacreation() -> str:
149
  try:
150
  log = run_notebook(NB1)
151
+ csvs = sorted(p.name for p in BASE_DIR.glob("*.csv"))
152
+ return f"OK{log}\n\nCSVs available:\n" + "\n".join(f" - {c}" for c in csvs)
153
  except Exception as e:
154
+ return f"FAILED{e}\n\n{traceback.format_exc()[-2000:]}"
 
155
 
156
  def run_pythonanalysis() -> str:
157
  try:
 
159
  idx = artifacts_index()
160
  figs = idx["python"]["figures"]
161
  tabs = idx["python"]["tables"]
162
+ return f"OK — {log}\n\nFigures: {', '.join(figs) or '(none)'}\nTables: {', '.join(tabs) or '(none)'}"
 
 
 
 
163
  except Exception as e:
164
+ return f"FAILED{e}\n\n{traceback.format_exc()[-2000:]}"
 
165
 
166
  def run_full_pipeline() -> str:
167
+ return "\n".join([
168
+ "=" * 58,
169
+ "STEP 1/2: Data Creation real-world PwC hospitality indicators",
170
+ "=" * 58,
171
+ run_datacreation(),
172
+ "",
173
+ "=" * 58,
174
+ "STEP 2/2: Python Analysis synthetic scores + dashboard artifacts",
175
+ "=" * 58,
176
+ run_pythonanalysis(),
177
+ ])
 
178
 
179
  # =========================================================
180
+ # DATA / TABLE LOADERS
181
  # =========================================================
182
 
183
+ def load_table_safe(path: Path) -> pd.DataFrame:
 
 
 
 
 
 
 
 
184
  try:
185
+ if path.suffix.lower() == ".json":
186
  obj = _read_json(path)
187
+ return pd.DataFrame([obj]) if isinstance(obj, dict) else pd.DataFrame(obj)
 
 
188
  return _read_csv(path)
189
  except Exception as e:
190
  return pd.DataFrame([{"error": str(e)}])
191
 
 
192
  def refresh_gallery():
193
+ figures = [(str(p), p.stem.replace("_", " ").title()) for p in sorted(PY_FIG_DIR.glob("*.png"))]
 
194
  idx = artifacts_index()
 
195
  table_choices = list(idx["python"]["tables"])
196
 
197
+ # Always include core datasets in table dropdown
198
+ for core in [CLEANED_CSV, ENRICHED_CSV]:
199
+ if (BASE_DIR / core).exists() and core not in table_choices:
200
+ table_choices.insert(0, core)
201
+
202
  default_df = pd.DataFrame()
203
  if table_choices:
204
+ chosen = table_choices[0]
205
+ path = BASE_DIR / chosen if (BASE_DIR / chosen).exists() else PY_TAB_DIR / chosen
206
+ default_df = load_table_safe(path)
 
 
 
 
207
 
208
+ return figures, gr.update(choices=table_choices, value=table_choices[0] if table_choices else None), default_df
209
 
210
  def on_table_select(choice: str):
211
  if not choice:
212
  return pd.DataFrame([{"hint": "Select a table above."}])
213
+ path = BASE_DIR / choice if (BASE_DIR / choice).exists() else PY_TAB_DIR / choice
214
  if not path.exists():
215
  return pd.DataFrame([{"error": f"File not found: {choice}"}])
216
+ return load_table_safe(path)
 
217
 
218
  # =========================================================
219
+ # KPIs
220
  # =========================================================
221
 
222
  def load_kpis() -> Dict[str, Any]:
223
+ df = read_market_data(prefer_enriched=True)
224
+ if df.empty:
225
+ return {}
226
+ syn = df[df["section"].eq("synthetic_features")]
227
+ risk = syn[syn["metric_name"].eq("risk_score")]
228
+ investment = syn[syn["metric_name"].eq("investment_potential_score")]
229
+ attractiveness = syn[syn["metric_name"].eq("market_attractiveness_score")]
230
+
231
+ return {
232
+ "locations": int(df["location"].nunique()),
233
+ "metrics": int(df["metric_name"].nunique()),
234
+ "real_rows": int((df["section"] != "synthetic_features").sum()),
235
+ "synthetic_rows": int((df["section"] == "synthetic_features").sum()),
236
+ "avg_risk_score": round(float(risk["value"].mean()), 1) if not risk.empty else None,
237
+ "avg_investment_potential": round(float(investment["value"].mean()), 1) if not investment.empty else None,
238
+ "avg_market_attractiveness": round(float(attractiveness["value"].mean()), 1) if not attractiveness.empty else None,
239
+ }
240
+
241
+ def render_kpi_cards() -> str:
242
+ kpis = load_kpis()
243
+ if not kpis:
244
+ return """
245
+ <div style='background:rgba(255,255,255,.72);border-radius:20px;padding:28px;text-align:center;border:1px solid #ddd;'>
246
+ <div style='font-size:34px;'>🏨</div>
247
+ <b>No hospitality data found yet</b><br>
248
+ <span>Run the pipeline or place the CSV files in the app folder.</span>
249
+ </div>"""
250
+
251
+ cards = [
252
+ ("📍", "Locations", kpis.get("locations"), "#a48de8"),
253
+ ("📊", "Metrics", kpis.get("metrics"), "#7aa6f8"),
254
+ ("🧪", "Synthetic Rows", kpis.get("synthetic_rows"), "#6ee7c7"),
255
+ ("⚠️", "Avg Risk Score", kpis.get("avg_risk_score"), "#e8537a"),
256
+ ("💼", "Avg Investment Potential", kpis.get("avg_investment_potential"), "#2ec4a0"),
257
+ ("⭐", "Avg Market Attractiveness", kpis.get("avg_market_attractiveness"), "#e8a230"),
258
+ ]
259
+
260
+ html = "<div style='display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin-bottom:24px;'>"
261
+ for icon, label, value, colour in cards:
262
+ value = "—" if value is None else value
263
+ html += f"""
264
+ <div style='background:rgba(255,255,255,.78);border-radius:20px;padding:18px;text-align:center;
265
+ border:1px solid rgba(124,92,191,.18);box-shadow:0 4px 16px rgba(124,92,191,.08);
266
+ border-top:3px solid {colour};'>
267
+ <div style='font-size:26px;margin-bottom:7px;'>{icon}</div>
268
+ <div style='color:#8b78c6;font-size:10px;text-transform:uppercase;letter-spacing:1.4px;font-weight:800;'>{label}</div>
269
+ <div style='color:#2d1f4e;font-size:18px;font-weight:800;margin-top:6px;'>{value}</div>
270
+ </div>"""
271
+ html += "</div>"
272
+ return html
273
+
274
+ # =========================================================
275
+ # INTERACTIVE CHARTS
276
+ # =========================================================
277
+
278
+ def styled_layout(**kwargs) -> dict:
279
+ defaults = dict(
280
+ template="plotly_white",
281
+ paper_bgcolor="rgba(255,255,255,0.96)",
282
+ plot_bgcolor="rgba(255,255,255,0.98)",
283
+ font=dict(family="system-ui, sans-serif", color="#2d1f4e", size=12),
284
+ margin=dict(l=60, r=20, t=70, b=70),
285
+ title=dict(font=dict(size=16, color="#4b2d8a")),
286
+ legend=dict(orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1),
287
+ )
288
+ defaults.update(kwargs)
289
+ return defaults
290
 
291
+ def empty_chart(title: str) -> go.Figure:
292
+ fig = go.Figure()
293
+ fig.update_layout(
294
+ title=title,
295
+ height=420,
296
+ template="plotly_white",
297
+ annotations=[dict(text="No data available yet", x=0.5, y=0.5, xref="paper", yref="paper", showarrow=False)],
298
+ )
299
+ return fig
300
+
301
+ def build_city_performance_chart() -> go.Figure:
302
+ df = read_market_data(True)
303
+ wide = pivot_metrics(df, section="city_performance", entity="city")
304
+ needed = [c for c in ["occupancy_yoy", "adr_yoy", "revpar_yoy"] if c in wide.columns]
305
+ if wide.empty or not needed:
306
+ return empty_chart("City Performance — Occupancy, ADR, RevPAR")
307
+ fig = go.Figure()
308
+ for i, col in enumerate(needed):
309
+ fig.add_trace(go.Bar(x=wide["location"], y=wide[col], name=col.replace("_", " ").upper(), marker_color=CHART_PALETTE[i]))
310
+ fig.update_layout(**styled_layout(title=dict(text="City Performance YoY — Occupancy, ADR, RevPAR"), barmode="group", height=430))
311
+ fig.update_yaxes(title="YoY change (%)")
312
+ return fig
313
+
314
+ def build_region_demand_chart() -> go.Figure:
315
+ df = read_market_data(True)
316
+ wide = pivot_metrics(df, section="regional_demand", entity="region")
317
+ needed = [c for c in ["domestic_demand_growth", "international_demand_growth"] if c in wide.columns]
318
+ if wide.empty or not needed:
319
+ return empty_chart("Regional Demand Growth")
320
+ fig = go.Figure()
321
+ for i, col in enumerate(needed):
322
+ fig.add_trace(go.Bar(y=wide["location"], x=wide[col], orientation="h", name=col.replace("_", " ").title(), marker_color=CHART_PALETTE[i]))
323
+ fig.update_layout(**styled_layout(title=dict(text="Regional Demand Growth — Domestic vs International"), barmode="group", height=max(420, len(wide)*42)))
324
+ fig.update_xaxes(title="Growth (%)")
325
+ fig.update_yaxes(autorange="reversed")
326
+ return fig
327
+
328
+ def build_synthetic_scores_chart() -> go.Figure:
329
+ df = read_market_data(True)
330
+ wide = pivot_metrics(df, section="synthetic_features")
331
+ score_cols = [c for c in ["growth_score", "market_attractiveness_score", "investment_potential_score", "risk_score"] if c in wide.columns]
332
+ if wide.empty or not score_cols:
333
+ return empty_chart("Synthetic Scores")
334
+ fig = go.Figure()
335
+ for i, col in enumerate(score_cols):
336
+ fig.add_trace(go.Bar(x=wide["location"], y=wide[col], name=col.replace("_", " ").title(), marker_color=CHART_PALETTE[i % len(CHART_PALETTE)]))
337
+ fig.update_layout(**styled_layout(title=dict(text="Synthetic Market Scores by Location"), barmode="group", height=470))
338
+ fig.update_yaxes(title="Score (0–100)", range=[0, 105])
339
+ return fig
340
+
341
+ def build_risk_chart() -> go.Figure:
342
+ df = read_market_data(True)
343
+ wide = pivot_metrics(df, section="synthetic_features")
344
+ if wide.empty or "risk_score" not in wide.columns:
345
+ return empty_chart("Risk Score")
346
+ wide = wide.sort_values("risk_score", ascending=True)
347
+ fig = go.Figure(go.Bar(
348
+ y=wide["location"],
349
+ x=wide["risk_score"],
350
+ orientation="h",
351
+ text=[f"{v:.1f}" for v in wide["risk_score"]],
352
+ marker=dict(color=wide["risk_score"], colorscale=[[0, "#2ec4a0"], [0.5, "#e8a230"], [1, "#e8537a"]]),
353
+ ))
354
+ fig.update_layout(**styled_layout(title=dict(text="Risk Score by Location"), showlegend=False, height=max(420, len(wide)*35)))
355
+ fig.update_xaxes(title="Risk score (0–100)", range=[0, 105])
356
+ return fig
357
+
358
+ def build_investment_chart() -> go.Figure:
359
+ df = read_market_data(True)
360
+ wide = pivot_metrics(df, section="synthetic_features")
361
+ if wide.empty or "investment_potential_score" not in wide.columns:
362
+ return empty_chart("Investment Potential")
363
+ wide = wide.sort_values("investment_potential_score", ascending=True)
364
+ fig = go.Figure(go.Bar(
365
+ y=wide["location"],
366
+ x=wide["investment_potential_score"],
367
+ orientation="h",
368
+ text=[f"{v:.1f}" for v in wide["investment_potential_score"]],
369
+ marker=dict(color=wide["investment_potential_score"], colorscale=[[0, "#c5b4f0"], [1, "#7c5cbf"]]),
370
+ ))
371
+ fig.update_layout(**styled_layout(title=dict(text="Investment Potential Score by Location"), showlegend=False, height=max(420, len(wide)*35)))
372
+ fig.update_xaxes(title="Investment potential score (0–100)", range=[0, 105])
373
+ return fig
374
+
375
+ def build_opportunity_table() -> pd.DataFrame:
376
+ df = read_market_data(True)
377
+ wide = pivot_metrics(df, section="synthetic_features")
378
+ if wide.empty:
379
+ return pd.DataFrame([{"hint": "No synthetic features found yet."}])
380
+ keep = [c for c in ["location", "entity", "growth_score", "market_attractiveness_score", "investment_potential_score", "risk_score", "risk_level", "opportunity_category"] if c in wide.columns]
381
+ out = wide[keep].copy()
382
+ for col in ["growth_score", "market_attractiveness_score", "investment_potential_score", "risk_score"]:
383
+ if col in out:
384
+ out[col] = out[col].round(1)
385
+ return out.sort_values(["entity", "investment_potential_score"], ascending=[True, False], na_position="last") if "investment_potential_score" in out else out
386
+
387
+ def refresh_dashboard():
388
+ figs, dd, df = refresh_gallery()
389
+ return (
390
+ render_kpi_cards(),
391
+ build_city_performance_chart(),
392
+ build_region_demand_chart(),
393
+ build_synthetic_scores_chart(),
394
+ build_risk_chart(),
395
+ build_investment_chart(),
396
+ build_opportunity_table(),
397
+ figs,
398
+ dd,
399
+ df,
400
+ )
401
 
402
  # =========================================================
403
+ # AI DASHBOARD N8N / HUGGING FACE / KEYWORD FALLBACK
404
  # =========================================================
405
 
406
+ DASHBOARD_SYSTEM = """You are the AI assistant for an Italy Hospitality Market Insight Assistant.
407
+ The app uses a real-world PwC Italy Hospitality Market Snapshot dataset and an enriched synthetic dataset.
408
+ The dataset has long-format columns: section, location, entity, metric_name, value, unit, period, source_page, note.
409
 
410
+ Available artifacts:
411
  {artifacts_json}
412
 
413
+ KPI summary:
414
+ {kpis_json}
415
+
416
+ Key concepts:
417
+ - city performance: occupancy_yoy, adr_yoy, revpar_yoy for Milan, Rome, Florence, Venice.
418
+ - regional demand: domestic_demand_growth and international_demand_growth.
419
+ - synthetic features: growth_score, market_attractiveness_score, investment_potential_score, risk_score, risk_level, opportunity_category.
420
+
421
+ Answer briefly and practically. At the END, output a JSON block exactly like:
422
+ ```json
423
+ {{"show": "chart"|"table"|"none", "chart": "city|region|scores|risk|investment|none", "table": "opportunities|raw|none"}}
424
+ ```
425
+ Choose:
426
+ - city for occupancy / ADR / RevPAR / city performance questions.
427
+ - region for domestic/international regional demand questions.
428
+ - scores for comparing synthetic scores.
429
+ - risk for risk score or risk level.
430
+ - investment for investment potential / attractiveness / opportunity.
431
+ - opportunities table for opportunity categories or strategic recommendations.
432
  """
433
 
434
  JSON_BLOCK_RE = re.compile(r"```json\s*(\{.*?\})\s*```", re.DOTALL)
435
  FALLBACK_JSON_RE = re.compile(r"\{[^{}]*\"show\"[^{}]*\}", re.DOTALL)
436
 
437
+ def parse_display_directive(text: str) -> Dict[str, str]:
438
+ for regex in [JSON_BLOCK_RE, FALLBACK_JSON_RE]:
439
+ m = regex.search(text)
440
+ if m:
441
+ try:
442
+ return json.loads(m.group(1) if regex is JSON_BLOCK_RE else m.group(0))
443
+ except Exception:
444
+ continue
445
+ return {"show": "none", "chart": "none", "table": "none"}
446
 
447
+ def clean_response(text: str) -> str:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
448
  return JSON_BLOCK_RE.sub("", text).strip()
449
 
450
+ def n8n_call(msg: str) -> Tuple[str, Optional[Dict[str, str]]]:
451
+ import requests
 
 
452
  try:
453
+ resp = requests.post(N8N_WEBHOOK_URL, json={"question": msg, "project": "italy_hospitality_market"}, timeout=25)
454
+ resp.raise_for_status()
455
  data = resp.json()
456
+ answer = data.get("answer") or data.get("reply") or "No answer returned by n8n."
457
  chart = data.get("chart", "none")
458
+ table = data.get("table", "none")
459
+ return answer, {"show": "chart" if chart != "none" else ("table" if table != "none" else "none"), "chart": chart, "table": table}
 
460
  except Exception as e:
461
+ return f"n8n error: {e}. Falling back to local logic.", None
462
+
463
+ def keyword_fallback(msg: str, kpis: Dict[str, Any]) -> Tuple[str, Dict[str, str]]:
464
+ m = msg.lower()
465
+ kpi_text = ""
466
+ if kpis:
467
+ kpi_text = f"The dataset covers {kpis.get('locations', '?')} locations and {kpis.get('metrics', '?')} metrics."
468
+
469
+ if any(w in m for w in ["occupancy", "adr", "revpar", "city", "milan", "rome", "florence", "venice"]):
470
+ return f"Here is the city performance view for occupancy, ADR, and RevPAR. {kpi_text}", {"show": "chart", "chart": "city", "table": "none"}
471
+ if any(w in m for w in ["region", "regional", "domestic", "international", "demand", "lazio", "puglia", "sicilia"]):
472
+ return f"Here is the regional demand comparison between domestic and international growth. {kpi_text}", {"show": "chart", "chart": "region", "table": "none"}
473
+ if any(w in m for w in ["risk", "risky", "safe", "low risk", "high risk"]):
474
+ return f"Here is the risk score view. Lower scores indicate safer or more stable opportunities. {kpi_text}", {"show": "chart", "chart": "risk", "table": "opportunities"}
475
+ if any(w in m for w in ["investment", "potential", "attractive", "attractiveness", "opportunity", "recommend"]):
476
+ return f"Here is the investment potential view, supported by the opportunity-category table. {kpi_text}", {"show": "chart", "chart": "investment", "table": "opportunities"}
477
+ if any(w in m for w in ["score", "scores", "growth", "synthetic", "compare"]):
478
+ return f"Here is the synthetic score comparison across locations. {kpi_text}", {"show": "chart", "chart": "scores", "table": "opportunities"}
479
+ if any(w in m for w in ["table", "data", "raw", "dataset"]):
480
+ return f"Here is the enriched hospitality dataset table preview. {kpi_text}", {"show": "table", "chart": "none", "table": "raw"}
481
+
482
+ return (
483
+ f"You can ask about city performance, regional demand, risk, investment potential, synthetic scores, or opportunity categories. {kpi_text}",
484
+ {"show": "none", "chart": "none", "table": "none"},
485
+ )
486
 
487
+ def resolve_chart(name: str):
488
+ return {
489
+ "city": build_city_performance_chart,
490
+ "region": build_region_demand_chart,
491
+ "scores": build_synthetic_scores_chart,
492
+ "risk": build_risk_chart,
493
+ "investment": build_investment_chart,
494
+ }.get(name, lambda: None)()
495
+
496
+ def resolve_table(name: str):
497
+ if name == "opportunities":
498
+ return build_opportunity_table()
499
+ if name == "raw":
500
+ return read_market_data(True).head(MAX_PREVIEW_ROWS)
501
+ return None
502
 
503
  def ai_chat(user_msg: str, history: list):
 
504
  if not user_msg or not user_msg.strip():
505
  return history, "", None, None
506
 
507
  idx = artifacts_index()
508
  kpis = load_kpis()
509
 
 
510
  if N8N_WEBHOOK_URL:
511
+ reply, directive = n8n_call(user_msg)
512
  if directive is None:
513
+ reply_fb, directive = keyword_fallback(user_msg, kpis)
514
+ reply = reply + "\n\n" + reply_fb
515
+ elif LLM_ENABLED:
 
 
516
  system = DASHBOARD_SYSTEM.format(
517
  artifacts_json=json.dumps(idx, indent=2),
518
+ kpis_json=json.dumps(kpis, indent=2) if kpis else "No KPIs available yet.",
519
  )
520
  msgs = [{"role": "system", "content": system}]
521
  for entry in (history or [])[-6:]:
522
  msgs.append(entry)
523
  msgs.append({"role": "user", "content": user_msg})
 
524
  try:
525
  r = llm_client.chat_completion(
526
  model=MODEL_NAME,
527
  messages=msgs,
528
+ temperature=0.25,
529
+ max_tokens=650,
530
  stream=False,
531
  )
532
+ raw = r["choices"][0]["message"]["content"] if isinstance(r, dict) else r.choices[0].message.content
533
+ directive = parse_display_directive(raw)
534
+ reply = clean_response(raw)
 
 
 
 
535
  except Exception as e:
536
+ reply_fb, directive = keyword_fallback(user_msg, kpis)
537
+ reply = f"Hugging Face error: {e}. Falling back to local logic.\n\n{reply_fb}"
538
+ else:
539
+ reply, directive = keyword_fallback(user_msg, kpis)
 
 
 
 
 
 
 
 
 
 
 
 
 
540
 
541
+ chart_out = resolve_chart(directive.get("chart", "none")) if directive.get("show") in ["chart", "figure"] or directive.get("chart") != "none" else None
542
+ table_out = resolve_table(directive.get("table", "none")) if directive.get("table") != "none" else None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
543
 
544
  new_history = (history or []) + [
545
  {"role": "user", "content": user_msg},
546
  {"role": "assistant", "content": reply},
547
  ]
548
+ return new_history, "", chart_out, table_out
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
549
 
550
  # =========================================================
551
  # UI
 
557
  css_path = BASE_DIR / "style.css"
558
  return css_path.read_text(encoding="utf-8") if css_path.exists() else ""
559
 
560
+ with gr.Blocks(title="Italy Hospitality Market Insight Assistant") as demo:
 
 
561
  gr.Markdown(
562
+ "# Italy Hospitality Market Insight Assistant\n"
563
+ "*A Gradio app for PwC-based hospitality indicators, synthetic market scores, n8n automation and Hugging Face AI Q&A.*",
564
  elem_id="escp_title",
565
  )
566
 
 
 
 
567
  with gr.Tab("Pipeline Runner"):
568
+ gr.Markdown("Run the project notebooks. Step 1 creates/cleans the dataset; Step 2 creates analysis outputs and synthetic insights.")
 
569
  with gr.Row():
570
+ btn_nb1 = gr.Button("Step 1: Data Creation", variant="secondary")
571
+ btn_nb2 = gr.Button("Step 2: Python Analysis", variant="secondary")
572
+ btn_all = gr.Button("Run Full Pipeline", variant="primary")
573
+ run_log = gr.Textbox(label="Execution Log", lines=18, max_lines=30, interactive=False)
 
 
 
 
 
 
 
 
 
 
 
574
  btn_nb1.click(run_datacreation, outputs=[run_log])
575
  btn_nb2.click(run_pythonanalysis, outputs=[run_log])
576
  btn_all.click(run_full_pipeline, outputs=[run_log])
577
 
 
 
 
578
  with gr.Tab("Dashboard"):
579
  kpi_html = gr.HTML(value=render_kpi_cards)
 
580
  refresh_btn = gr.Button("Refresh Dashboard", variant="primary")
581
 
582
+ gr.Markdown("#### Interactive Hospitality Charts")
583
+ with gr.Row():
584
+ chart_city = gr.Plot(label="City Performance")
585
+ chart_region = gr.Plot(label="Regional Demand")
586
+ with gr.Row():
587
+ chart_scores = gr.Plot(label="Synthetic Scores")
588
+ chart_risk = gr.Plot(label="Risk Score")
589
+ chart_investment = gr.Plot(label="Investment Potential")
 
 
 
 
590
 
591
+ gr.Markdown("#### Opportunity Categories")
592
+ opportunity_table = gr.Dataframe(label="Synthetic Strategy Table", interactive=False)
 
 
 
 
 
 
 
 
593
 
594
+ gr.Markdown("#### Static Figures and Data Tables")
595
+ gallery = gr.Gallery(label="Generated Figures", columns=2, height=430, object_fit="contain")
596
+ table_dropdown = gr.Dropdown(label="Select a table to view", choices=[], interactive=True)
597
+ table_display = gr.Dataframe(label="Table Preview", interactive=False)
598
 
599
  refresh_btn.click(
600
+ refresh_dashboard,
601
+ outputs=[kpi_html, chart_city, chart_region, chart_scores, chart_risk, chart_investment, opportunity_table, gallery, table_dropdown, table_display],
 
 
 
 
 
 
602
  )
603
+ table_dropdown.change(on_table_select, inputs=[table_dropdown], outputs=[table_display])
604
 
 
 
 
605
  with gr.Tab('"AI" Dashboard'):
606
+ ai_status = (
607
+ "Connected to your **n8n workflow**." if N8N_WEBHOOK_URL else
608
+ "**Hugging Face LLM active.**" if LLM_ENABLED else
609
+ "Using **local keyword matching**. To activate AI, set `HF_API_KEY`; to activate automations, set `N8N_WEBHOOK_URL`."
 
 
610
  )
611
  gr.Markdown(
612
+ "### Ask questions about the Italy hospitality market\n\n"
613
+ f"{ai_status}\n\n"
614
+ "Examples: *Which city has the strongest RevPAR?*, *Show risk scores*, *Which regions are investment opportunities?*"
615
  )
 
616
  with gr.Row(equal_height=True):
617
  with gr.Column(scale=1):
618
+ chatbot = gr.Chatbot(label="Conversation", height=400, type="messages")
619
+ user_input = gr.Textbox(label="Ask about your data", placeholder="e.g. Show me investment potential by location", lines=1)
 
 
 
 
 
 
 
620
  gr.Examples(
621
  examples=[
622
+ "Show me city performance for occupancy, ADR and RevPAR",
623
+ "Which locations have the highest risk?",
624
+ "Show investment potential by location",
625
+ "Compare synthetic scores",
626
+ "What is regional domestic vs international demand?",
627
+ "Give me strategic recommendations",
628
  ],
629
  inputs=user_input,
630
  )
 
631
  with gr.Column(scale=1):
632
+ ai_figure = gr.Plot(label="Interactive Chart")
633
+ ai_table = gr.Dataframe(label="Relevant Table", interactive=False)
 
 
 
 
 
 
 
 
 
 
 
634
 
635
+ user_input.submit(ai_chat, inputs=[user_input, chatbot], outputs=[chatbot, user_input, ai_figure, ai_table])
636
 
637
+ if __name__ == "__main__":
638
+ demo.launch(css=load_css(), allowed_paths=[str(BASE_DIR)])