Kogann commited on
Commit
325bba9
Β·
verified Β·
1 Parent(s): 62d2692

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +421 -105
app.py CHANGED
@@ -6,19 +6,23 @@ Pipeline
6
  1. load + normalize deterministic
7
  2. context card deterministic
8
  3. code generation LLM
9
- 4. sandboxed execution deterministic
10
  5. one repair pass LLM, given a cleaned traceback
11
- 6. facts + tables deterministic (argmax, describe, corr)
12
  7. narrative LLM, one sentence per pre-computed fact
13
 
14
  Design principle: the model does the two things it is good at β€” writing analysis
15
  code and repairing it. Every lookup, table and superlative is computed in Python.
 
 
 
16
  """
17
 
18
  import base64
19
  import contextlib
20
  import glob
21
  import io
 
22
  import os
23
  import re
24
  import shutil
@@ -34,6 +38,8 @@ import torch
34
  matplotlib.use("Agg")
35
  import matplotlib.pyplot as plt # noqa: E402
36
  import seaborn as sns # noqa: E402
 
 
37
  from huggingface_hub import HfApi, hf_hub_download # noqa: E402
38
  from transformers import AutoModelForCausalLM, AutoTokenizer # noqa: E402
39
 
@@ -42,11 +48,18 @@ from transformers import AutoModelForCausalLM, AutoTokenizer # noqa: E402
42
  # --------------------------------------------------------------------------- #
43
  MODEL_ID = "Qwen/Qwen2.5-Coder-1.5B-Instruct"
44
 
 
 
 
 
 
 
45
  tok = AutoTokenizer.from_pretrained(MODEL_ID)
46
- model = AutoModelForCausalLM.from_pretrained(MODEL_ID, dtype=torch.bfloat16).to("cuda")
47
  model.eval()
48
  if tok.pad_token_id is None:
49
  tok.pad_token = tok.eos_token
 
50
 
51
  PLOTS_DIR = "plots"
52
 
@@ -121,6 +134,15 @@ def normalize(df: pd.DataFrame) -> pd.DataFrame:
121
  return df
122
 
123
 
 
 
 
 
 
 
 
 
 
124
  # --------------------------------------------------------------------------- #
125
  # 2. Context card
126
  # --------------------------------------------------------------------------- #
@@ -173,7 +195,7 @@ def extract_code(text: str) -> str:
173
 
174
 
175
  # --------------------------------------------------------------------------- #
176
- # 4. Execution sandbox
177
  # --------------------------------------------------------------------------- #
178
  BANNED = re.compile(r"\b(pd\.read_\w+|load_dataset|sns\.load_dataset)\s*\(")
179
  CORR_FIX = re.compile(r"\.corr\(\s*\)") # bare df.corr() -> numeric_only
@@ -185,6 +207,9 @@ UNSAFE = re.compile(
185
  r"\bimport\s+(os|sys)\b|\bopen\s*\(|\beval\s*\(|\bexec\s*\(|\bgetattr\s*\()"
186
  )
187
 
 
 
 
188
 
189
  def sanitize(code: str) -> str:
190
  """Neutralize data reloads; patch the one error the model cannot reliably fix.
@@ -202,13 +227,89 @@ def sanitize(code: str) -> str:
202
  return "\n".join(out)
203
 
204
 
205
- def drop_index_cols(df: pd.DataFrame):
206
- """Remove monotonic unique integer columns β€” row ids, not variables."""
207
- drop = [c for c in df.columns
208
- if pd.api.types.is_integer_dtype(df[c])
209
- and df[c].is_monotonic_increasing
210
- and df[c].nunique() == len(df)]
211
- return df.drop(columns=drop), drop
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
212
 
213
 
214
  def run_code(code: str, df: pd.DataFrame) -> dict:
@@ -218,20 +319,22 @@ def run_code(code: str, df: pd.DataFrame) -> dict:
218
 
219
  before = set(glob.glob(f"{PLOTS_DIR}/*.png"))
220
  ns = {"df": df.copy(), "pd": pd, "np": np, "plt": plt, "sns": sns}
 
 
221
  buf, err = io.StringIO(), None
222
  try:
223
  with contextlib.redirect_stdout(buf):
224
  exec(code, ns) # noqa: S102
225
  except Exception:
226
  err = traceback.format_exc(limit=3)
227
-
228
- # safety net: model called plt.show(), or crashed with a figure still open
229
- for i, num in enumerate(plt.get_fignums(), start=1):
230
- fig = plt.figure(num)
231
- if not fig.get_axes(): # skip blank figures
232
- continue
233
- fig.savefig(f"{PLOTS_DIR}/figure_{i}.png", bbox_inches="tight")
234
- plt.close("all")
235
 
236
  return {"ok": err is None, "stdout": buf.getvalue(), "error": err,
237
  "plots": sorted(set(glob.glob(f"{PLOTS_DIR}/*.png")) - before)}
@@ -254,6 +357,18 @@ def format_error(err: str, code: str) -> str:
254
  REDUNDANT_R = 0.99 # |r| at or above this is a duplicate encoding, not a finding
255
 
256
 
 
 
 
 
 
 
 
 
 
 
 
 
257
  def corr_pairs(df: pd.DataFrame):
258
  """Split correlations into real findings and near-duplicate columns."""
259
  num = df.select_dtypes(include="number")
@@ -273,13 +388,20 @@ def corr_pairs(df: pd.DataFrame):
273
 
274
 
275
  def make_tables(df: pd.DataFrame, max_card: int = 6) -> dict:
 
 
 
 
 
276
  t = {}
 
277
  n = df.isna().sum()
278
  n = n[n > 0].sort_values(ascending=False)
279
  if len(n):
280
  head = n.head(15)
281
- tbl = pd.DataFrame({"missing": head,
282
- "pct": (100 * head / len(df)).round(1)}).to_markdown()
 
283
  if len(n) > 15:
284
  tbl += f"\n\n_+{len(n) - 15} more columns with missing values._"
285
  t["missing"] = tbl
@@ -305,8 +427,12 @@ def make_tables(df: pd.DataFrame, max_card: int = 6) -> dict:
305
  + pd.DataFrame(dup[:8]).to_markdown(index=False))
306
 
307
  num = df.select_dtypes(include="number")
308
- t["describe"] = (num.describe().T.round(2).to_markdown() # transposed
309
- if len(num.columns) else "_None._")
 
 
 
 
310
  return t
311
 
312
 
@@ -318,10 +444,10 @@ def key_facts(df: pd.DataFrame, max_card: int = 6) -> dict:
318
  if len(n):
319
  c = n.idxmax()
320
  f["missing"] = (f"{c} has the most missing values: "
321
- f"{n.max()} ({100 * n.max() / len(df):.1f}%)")
322
 
323
- bins, best = [c for c in df.select_dtypes(include="number")
324
- if df[c].nunique() == 2], None
325
  for c in [c for c in df.columns if df[c].nunique(dropna=True) <= max_card]:
326
  for b in bins:
327
  if b == c:
@@ -358,11 +484,12 @@ def one_liner(fact: str) -> str:
358
 
359
 
360
  def audit_numbers(md: str, allowed_text: str) -> list:
361
- """Flag numbers in the prose absent from the evidence. Detects fabrication,
362
  not misinterpretation β€” a flag list, not a verdict."""
363
- allowed = set(re.findall(r"\d+\.?\d*", allowed_text))
364
  prose = "\n".join(l for l in md.splitlines() if not l.strip().startswith("|"))
365
- return sorted({x for x in re.findall(r"\d+\.?\d*", prose) if x not in allowed})
 
366
 
367
 
368
  def embed_images(md: str) -> str:
@@ -376,46 +503,143 @@ def embed_images(md: str) -> str:
376
  return re.sub(r"\]\(([^)]+\.png)\)", repl, md)
377
 
378
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
379
  # --------------------------------------------------------------------------- #
380
- # 7. Gradio handler
381
  # --------------------------------------------------------------------------- #
382
  DATASET_RE = re.compile(r"^[\w.-]+(/[\w.-]+)?$")
383
- BLANK = ("", "", "", [], "", "")
384
 
385
 
386
- @spaces.GPU(duration=150)
387
- def run_agent(instruction: str, dataset: str):
388
- """Streams: status, code, stdout, gallery, report, flags."""
389
  instruction = (instruction or "").strip()
390
  dataset = (dataset or "").strip()
391
  if not DATASET_RE.match(dataset):
392
- yield ("**Invalid dataset id.** Use the form `owner/name`.", *BLANK[1:])
393
  return
394
 
395
  shutil.rmtree(PLOTS_DIR, ignore_errors=True)
396
  os.makedirs(PLOTS_DIR, exist_ok=True)
397
 
398
- yield (f"Loading `{dataset}` …", "", "", [], "", "")
399
  try:
400
  df = normalize(load_hf_dataframe(dataset))
401
  except Exception as e:
402
- yield (f"**Could not load `{dataset}`.** {type(e).__name__}: {e}", *BLANK[1:])
403
  return
404
 
405
  df, dropped = drop_index_cols(df)
406
  context = build_context(df, dataset)
407
- yield (f"Loaded **{df.shape[0]} x {df.shape[1]}**. Generating analysis code …",
408
- "", "", [], "", "")
 
409
 
410
  code = sanitize(extract_code(
411
  generate(CODE_SYSTEM, f"{context}\n\nTask: {instruction}", max_new_tokens=1200)))
412
  res = run_code(code, df)
413
  yield (f"Attempt 1: {'ok' if res['ok'] else 'failed'}. Executing …",
414
- code, res["stdout"], res["plots"], "", "")
415
 
416
  if not res["ok"]:
417
  yield ("Attempt 1 failed β€” repairing from the traceback …",
418
- code, res["stdout"] + "\n" + format_error(res["error"], code),
419
  res["plots"], "", "")
420
  fix = (f"{context}\n\nThis code failed:\n```python\n{code}\n```\n\n"
421
  f"Error:\n{format_error(res['error'], code)}\n\nReturn the corrected script.")
@@ -423,99 +647,191 @@ def run_agent(instruction: str, dataset: str):
423
  res = run_code(code, df)
424
 
425
  plots = sorted(glob.glob(f"{PLOTS_DIR}/*.png"))
 
426
  status = ("Code ran successfully" if res["ok"]
427
  else "Code still failing after one repair β€” report built from data only")
428
- yield (f"{status}. Writing report …", code, res["stdout"], plots, "", "")
429
 
430
- t, kf = make_tables(df), key_facts(df)
431
- P = [f"# EDA Report β€” {dataset}", "", f"*{instruction}*", "",
432
- "## Overview", one_liner(schema_only(df, dataset)), ""]
433
- if dropped:
434
- P += [f"_Index-like columns excluded from analysis: {', '.join(dropped)}._", ""]
435
- P += ["## Missing Values", t["missing"], ""]
436
- if "missing" in kf:
437
- P += [one_liner(kf["missing"]), ""]
438
- P += ["## Group Differences", t["groups"], ""]
439
- if "group" in kf:
440
- P += [one_liner(kf["group"]), ""]
441
- P += ["## Correlations", t["corr"], ""]
442
- if "corr" in kf:
443
- P += [one_liner(kf["corr"]), ""]
444
- if "redundant" in t:
445
- P += ["## Redundant Columns", t["redundant"], ""]
446
- P += ["## Numeric Summary", t["describe"], ""]
447
 
448
- take = generate("Rewrite each fact as one markdown bullet. Add nothing.",
449
- "\n".join(f"- {v}" for v in kf.values()), max_new_tokens=200)
450
- P += ["## Takeaways",
451
- *[l.strip() for l in take.splitlines() if l.strip().startswith(("-", "*"))][:3],
452
- ""]
453
 
454
- P += ["## Figures", ""]
455
- for p in plots:
456
- P += [f"**{os.path.basename(p)[:-4].replace('_', ' ')}**", "", f"![]({p})", ""]
457
 
458
- report = "\n".join(P)
459
- flags = audit_numbers(report, "\n".join(t.values()) + "\n".join(kf.values())
460
- + schema_only(df, dataset))
 
 
 
 
461
 
462
- with open("eda_report.md", "w") as fh:
463
- fh.write(report)
 
 
 
 
 
 
 
 
 
464
 
465
- yield (f"Done β€” {status.lower()}, {len(plots)} figures.",
466
- code, res["stdout"], plots, embed_images(report),
467
- ", ".join(flags) if flags else "none")
 
 
 
468
 
469
 
 
 
 
470
  DEFAULT_INSTRUCTION = ("Run a comprehensive exploratory data analysis, highlighting "
471
  "missing values, distributions, and key feature correlations.")
472
 
473
- with gr.Blocks(title="EDA Agent") as demo:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
474
  gr.Markdown(
475
- "# EDA Agent\n"
476
- "A **Qwen2.5-Coder-1.5B** agent that writes its own analysis code, executes it, "
477
- "repairs it from the traceback if it crashes, and reports the results.\n\n"
478
- "Tables, figures and every superlative are computed in Python β€” the model writes "
479
- "code and prose only. Runs on ZeroGPU; a full report takes about a minute."
480
  )
481
 
 
 
 
 
 
 
 
 
 
 
482
  with gr.Row():
483
  instruction = gr.Textbox(label="Prompt instruction", value=DEFAULT_INSTRUCTION,
484
  lines=3, scale=3)
485
- dataset = gr.Textbox(label="Hugging Face dataset", value="mstz/titanic",
486
- lines=1, scale=1)
487
- run_btn = gr.Button("Run EDA Agent", variant="primary")
 
 
 
 
 
 
 
 
 
488
  status = gr.Markdown()
 
489
 
490
  with gr.Tabs():
491
- with gr.Tab("Report"):
492
  report_md = gr.Markdown()
493
  flags_box = gr.Textbox(label="Unverified numbers (fabrication check)",
494
  interactive=False)
495
- with gr.Tab("Generated code"):
 
 
496
  code_box = gr.Code(language="python", label="Model-written analysis code")
497
- with gr.Tab("Execution output"):
498
  stdout_box = gr.Textbox(label="stdout / traceback", lines=18,
499
  interactive=False)
500
- with gr.Tab("Plots"):
501
- gallery = gr.Gallery(label="Figures", columns=2, height=520)
502
-
503
- gr.Examples(
504
- examples=[
505
- [DEFAULT_INSTRUCTION, "mstz/titanic"],
506
- ["Explore this dataset: missing values, distributions, correlations.",
507
- "scikit-learn/iris"],
508
- ["Summarise the distributions and flag any strongly correlated features.",
509
- "scikit-learn/adult-census-income"],
510
- ],
511
- inputs=[instruction, dataset],
512
- )
513
 
514
- run_btn.click(
515
- run_agent,
516
- inputs=[instruction, dataset],
517
- outputs=[status, code_box, stdout_box, gallery, report_md, flags_box],
518
- )
 
 
519
 
520
  if __name__ == "__main__":
521
  demo.queue().launch()
 
6
  1. load + normalize deterministic
7
  2. context card deterministic
8
  3. code generation LLM
9
+ 4. sandboxed execution deterministic (+ plot-quality guards)
10
  5. one repair pass LLM, given a cleaned traceback
11
+ 6. facts + tables deterministic (idxmax, describe, corr)
12
  7. narrative LLM, one sentence per pre-computed fact
13
 
14
  Design principle: the model does the two things it is good at β€” writing analysis
15
  code and repairing it. Every lookup, table and superlative is computed in Python.
16
+
17
+ The three Quick Starters are served from a pre-generated cache in quickstarts/
18
+ and never touch the model, so the common path is instant and costs no GPU quota.
19
  """
20
 
21
  import base64
22
  import contextlib
23
  import glob
24
  import io
25
+ import json
26
  import os
27
  import re
28
  import shutil
 
38
  matplotlib.use("Agg")
39
  import matplotlib.pyplot as plt # noqa: E402
40
  import seaborn as sns # noqa: E402
41
+ from matplotlib.ticker import (FuncFormatter, LogLocator, # noqa: E402
42
+ NullFormatter, ScalarFormatter)
43
  from huggingface_hub import HfApi, hf_hub_download # noqa: E402
44
  from transformers import AutoModelForCausalLM, AutoTokenizer # noqa: E402
45
 
 
48
  # --------------------------------------------------------------------------- #
49
  MODEL_ID = "Qwen/Qwen2.5-Coder-1.5B-Instruct"
50
 
51
+ # On ZeroGPU, `spaces` enables CUDA emulation at import, so placing the model on
52
+ # "cuda" at module level is required. On CPU Spaces (or locally) there is no CUDA,
53
+ # so fall back to float32 on CPU β€” the app boots, but generation is far slower.
54
+ DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
55
+ DTYPE = torch.bfloat16 if DEVICE == "cuda" else torch.float32
56
+
57
  tok = AutoTokenizer.from_pretrained(MODEL_ID)
58
+ model = AutoModelForCausalLM.from_pretrained(MODEL_ID, dtype=DTYPE).to(DEVICE)
59
  model.eval()
60
  if tok.pad_token_id is None:
61
  tok.pad_token = tok.eos_token
62
+ print(f"[startup] device={DEVICE} dtype={DTYPE}")
63
 
64
  PLOTS_DIR = "plots"
65
 
 
134
  return df
135
 
136
 
137
+ def drop_index_cols(df: pd.DataFrame):
138
+ """Remove monotonic unique integer columns β€” row ids, not variables."""
139
+ drop = [c for c in df.columns
140
+ if pd.api.types.is_integer_dtype(df[c])
141
+ and df[c].is_monotonic_increasing
142
+ and df[c].nunique() == len(df)]
143
+ return df.drop(columns=drop), drop
144
+
145
+
146
  # --------------------------------------------------------------------------- #
147
  # 2. Context card
148
  # --------------------------------------------------------------------------- #
 
195
 
196
 
197
  # --------------------------------------------------------------------------- #
198
+ # 4. Execution sandbox (+ plot-quality guards)
199
  # --------------------------------------------------------------------------- #
200
  BANNED = re.compile(r"\b(pd\.read_\w+|load_dataset|sns\.load_dataset)\s*\(")
201
  CORR_FIX = re.compile(r"\.corr\(\s*\)") # bare df.corr() -> numeric_only
 
207
  r"\bimport\s+(os|sys)\b|\bopen\s*\(|\beval\s*\(|\bexec\s*\(|\bgetattr\s*\()"
208
  )
209
 
210
+ MAX_ANNOT_COLS = 12 # above this, heatmap cell numbers are unreadable
211
+ MAX_TICKS = 20 # above this, thin the tick labels
212
+
213
 
214
  def sanitize(code: str) -> str:
215
  """Neutralize data reloads; patch the one error the model cannot reliably fix.
 
227
  return "\n".join(out)
228
 
229
 
230
+ def _short(s, n=18):
231
+ s = str(s)
232
+ return s if len(s) <= n else f"{s[:n//2 - 1]}…{s[-(n//2 - 1):]}"
233
+
234
+
235
+ def _tick(v, _=None):
236
+ """Plain numbers with thousands separators β€” never 1e7."""
237
+ return f"{v:,.0f}" if abs(v) >= 1000 else f"{v:g}"
238
+
239
+
240
+ def _tidy(fig):
241
+ """Make any figure legible: thin dense ticks, rotate, kill scientific notation."""
242
+ for ax in fig.get_axes():
243
+ if len(ax.get_xticklabels()) > MAX_TICKS:
244
+ step = max(1, len(ax.get_xticks()) // MAX_TICKS)
245
+ ax.set_xticks(ax.get_xticks()[::step])
246
+ for axis, scale in ((ax.xaxis, ax.get_xscale()), (ax.yaxis, ax.get_yscale())):
247
+ # only touch numeric axes: a heatmap's labels use a FixedFormatter
248
+ if scale == "linear" and isinstance(axis.get_major_formatter(), ScalarFormatter):
249
+ axis.set_major_formatter(FuncFormatter(_tick))
250
+ plt.setp(ax.get_xticklabels(), rotation=45, ha="right", fontsize=8)
251
+ plt.setp(ax.get_yticklabels(), fontsize=8)
252
+
253
+
254
+ def _install_plot_guards():
255
+ """Patch the real modules, so `import matplotlib.pyplot as plt` cannot bypass us."""
256
+ real = {"savefig": plt.savefig, "heatmap": sns.heatmap, "histplot": sns.histplot}
257
+
258
+ def savefig(fname, *a, **kw):
259
+ fig = plt.gcf()
260
+ w, h = fig.get_size_inches()
261
+ fig.set_size_inches(max(w, 8), max(h, 5))
262
+ _tidy(fig)
263
+ kw.setdefault("bbox_inches", "tight")
264
+ kw.setdefault("dpi", 110)
265
+ return real["savefig"](fname, *a, **kw)
266
+
267
+ def heatmap(data, *a, **kw):
268
+ n = getattr(data, "shape", (0, 0))[1]
269
+ if n > MAX_ANNOT_COLS:
270
+ kw["annot"] = False
271
+ kw.setdefault("cmap", "coolwarm")
272
+ side = min(max(7, 0.5 * n + 4), 20)
273
+ plt.gcf().set_size_inches(side, side * 0.85)
274
+ ax = real["heatmap"](data, *a, **kw)
275
+ if hasattr(data, "columns"):
276
+ fs = 8 if n <= 15 else 6
277
+ ax.set_xticklabels([_short(c, 24) for c in data.columns],
278
+ rotation=45, ha="right", fontsize=fs)
279
+ ax.set_yticklabels([_short(c, 24) for c in data.index],
280
+ rotation=0, fontsize=fs)
281
+ return ax
282
+
283
+ def _series(a, kw):
284
+ d, x = kw.get("data", a[0] if a else None), kw.get("x")
285
+ try:
286
+ if isinstance(x, str) and hasattr(d, "columns"):
287
+ return pd.to_numeric(d[x], errors="coerce").dropna()
288
+ if isinstance(d, pd.Series):
289
+ return pd.to_numeric(d, errors="coerce").dropna()
290
+ except Exception:
291
+ pass
292
+ return None
293
+
294
+ def histplot(*a, **kw):
295
+ s = _series(a, kw)
296
+ logged = False
297
+ # heavy right skew (prices, fares, incomes) -> log x, else one tall bar
298
+ if (s is not None and len(s) > 20 and s.min() > 0
299
+ and s.max() / max(s.median(), 1e-9) > 50 and "log_scale" not in kw):
300
+ kw["log_scale"] = (True, False)
301
+ logged = True
302
+ ax = real["histplot"](*a, **kw)
303
+ if logged:
304
+ ax.xaxis.set_major_locator(LogLocator(base=10, subs=(1.0, 2.0, 5.0),
305
+ numticks=12))
306
+ ax.xaxis.set_major_formatter(FuncFormatter(_tick))
307
+ ax.xaxis.set_minor_formatter(NullFormatter())
308
+ ax.set_xlabel(f"{ax.get_xlabel()} (log scale)")
309
+ return ax
310
+
311
+ plt.savefig, sns.heatmap, sns.histplot = savefig, heatmap, histplot
312
+ return real
313
 
314
 
315
  def run_code(code: str, df: pd.DataFrame) -> dict:
 
319
 
320
  before = set(glob.glob(f"{PLOTS_DIR}/*.png"))
321
  ns = {"df": df.copy(), "pd": pd, "np": np, "plt": plt, "sns": sns}
322
+
323
+ real = _install_plot_guards()
324
  buf, err = io.StringIO(), None
325
  try:
326
  with contextlib.redirect_stdout(buf):
327
  exec(code, ns) # noqa: S102
328
  except Exception:
329
  err = traceback.format_exc(limit=3)
330
+ finally:
331
+ for i, num in enumerate(plt.get_fignums(), start=1):
332
+ fig = plt.figure(num)
333
+ if fig.get_axes():
334
+ fig.savefig(f"{PLOTS_DIR}/figure_{i}.png")
335
+ plt.close("all")
336
+ plt.savefig, sns.heatmap, sns.histplot = (
337
+ real["savefig"], real["heatmap"], real["histplot"])
338
 
339
  return {"ok": err is None, "stdout": buf.getvalue(), "error": err,
340
  "plots": sorted(set(glob.glob(f"{PLOTS_DIR}/*.png")) - before)}
 
357
  REDUNDANT_R = 0.99 # |r| at or above this is a duplicate encoding, not a finding
358
 
359
 
360
+ def fmt_num(v):
361
+ """Plain, comma-separated numbers. Never 1.02011e+06."""
362
+ if pd.isna(v):
363
+ return ""
364
+ v = float(v)
365
+ if v.is_integer():
366
+ return f"{int(v):,}"
367
+ if abs(v) >= 1000:
368
+ return f"{v:,.0f}"
369
+ return f"{v:,.2f}"
370
+
371
+
372
  def corr_pairs(df: pd.DataFrame):
373
  """Split correlations into real findings and near-duplicate columns."""
374
  num = df.select_dtypes(include="number")
 
388
 
389
 
390
  def make_tables(df: pd.DataFrame, max_card: int = 6) -> dict:
391
+ """Every table rendered by pandas. The model never transcribes one.
392
+
393
+ disable_numparse stops tabulate re-parsing our formatted strings and
394
+ re-rendering large values in scientific notation.
395
+ """
396
  t = {}
397
+
398
  n = df.isna().sum()
399
  n = n[n > 0].sort_values(ascending=False)
400
  if len(n):
401
  head = n.head(15)
402
+ tbl = pd.DataFrame({"missing": head.map(fmt_num),
403
+ "pct": (100 * head / len(df)).round(1)}
404
+ ).to_markdown(disable_numparse=True)
405
  if len(n) > 15:
406
  tbl += f"\n\n_+{len(n) - 15} more columns with missing values._"
407
  t["missing"] = tbl
 
427
  + pd.DataFrame(dup[:8]).to_markdown(index=False))
428
 
429
  num = df.select_dtypes(include="number")
430
+ if len(num.columns):
431
+ desc = num.describe().T
432
+ desc = desc.map(fmt_num) if hasattr(desc, "map") else desc.applymap(fmt_num)
433
+ t["describe"] = desc.to_markdown(disable_numparse=True)
434
+ else:
435
+ t["describe"] = "_No numeric columns._"
436
  return t
437
 
438
 
 
444
  if len(n):
445
  c = n.idxmax()
446
  f["missing"] = (f"{c} has the most missing values: "
447
+ f"{fmt_num(n.max())} ({100 * n.max() / len(df):.1f}%)")
448
 
449
+ bins = [c for c in df.select_dtypes(include="number") if df[c].nunique() == 2]
450
+ best = None
451
  for c in [c for c in df.columns if df[c].nunique(dropna=True) <= max_card]:
452
  for b in bins:
453
  if b == c:
 
484
 
485
 
486
  def audit_numbers(md: str, allowed_text: str) -> list:
487
+ """Flag numerals in the prose absent from the evidence. Detects fabrication,
488
  not misinterpretation β€” a flag list, not a verdict."""
489
+ allowed = set(re.findall(r"\d+\.?\d*", allowed_text.replace(",", "")))
490
  prose = "\n".join(l for l in md.splitlines() if not l.strip().startswith("|"))
491
+ return sorted({x for x in re.findall(r"\d+\.?\d*", prose.replace(",", ""))
492
+ if x not in allowed})
493
 
494
 
495
  def embed_images(md: str) -> str:
 
503
  return re.sub(r"\]\(([^)]+\.png)\)", repl, md)
504
 
505
 
506
+ def report_to_pdf(report_md: str, plots: list, path: str = "eda_report.pdf") -> str:
507
+ """Render the markdown report plus every figure to a single PDF."""
508
+ from fpdf import FPDF
509
+ from fpdf.enums import XPos, YPos
510
+
511
+ def clean(s):
512
+ for a, b in [("β€”", "-"), ("…", "..."), ("β‰₯", ">="), ("Γ—", "x"),
513
+ ("’", "'"), ("β€œ", '"'), ("”", '"')]:
514
+ s = s.replace(a, b)
515
+ return s.encode("latin-1", "replace").decode("latin-1")
516
+
517
+ pdf = FPDF(format="A4")
518
+ pdf.set_auto_page_break(True, margin=15)
519
+ pdf.add_page()
520
+ usable = pdf.w - pdf.l_margin - pdf.r_margin
521
+ max_table_chars = int(usable / (6.5 * 0.6 * 0.3528)) # Courier 0.6 em, 1pt=0.3528mm
522
+
523
+ def write(text, style="", size=10, h=5):
524
+ """Always start at the left margin β€” otherwise multi_cell width goes to zero."""
525
+ pdf.set_font("Courier" if style == "mono" else "Helvetica",
526
+ "B" if style == "B" else "", size)
527
+ pdf.set_x(pdf.l_margin)
528
+ pdf.multi_cell(0, h, text, new_x=XPos.LMARGIN, new_y=YPos.NEXT)
529
+
530
+ for raw in report_md.splitlines():
531
+ line = clean(raw.rstrip())
532
+ if line.startswith("!["):
533
+ continue
534
+ is_table = line.startswith("|")
535
+ if not is_table:
536
+ line = re.sub(r"[*_`]", "", line)
537
+ line = " ".join(w if len(w) <= 50 else
538
+ " ".join(w[i:i + 50] for i in range(0, len(w), 50))
539
+ for w in line.split())
540
+ elif len(line) > max_table_chars:
541
+ line = line[:max_table_chars - 3] + "..."
542
+
543
+ if line.startswith("# "):
544
+ write(line[2:], "B", 15, 8); pdf.ln(1)
545
+ elif line.startswith("## "):
546
+ pdf.ln(2); write(line[3:], "B", 12, 7)
547
+ elif is_table:
548
+ write(line, "mono", 6.5, 3.4)
549
+ elif line.strip():
550
+ write(line, "", 10, 5)
551
+ else:
552
+ pdf.ln(2)
553
+
554
+ for p in plots:
555
+ pdf.add_page()
556
+ write(clean(os.path.basename(p)[:-4].replace("_", " ")), "B", 11, 7)
557
+ pdf.image(p, w=usable)
558
+
559
+ pdf.output(path)
560
+ return path
561
+
562
+
563
+ def build_report(df, res, name, instruction, dropped=None) -> tuple:
564
+ t, kf = make_tables(df), key_facts(df)
565
+
566
+ P = [f"# EDA Report β€” {name}", "", f"*{instruction}*", "",
567
+ "## Overview", one_liner(schema_only(df, name)), ""]
568
+ if dropped:
569
+ P += [f"_Index-like columns excluded from analysis: {', '.join(dropped)}._", ""]
570
+
571
+ P += ["## Missing Values", t["missing"], ""]
572
+ if "missing" in kf:
573
+ P += [one_liner(kf["missing"]), ""]
574
+
575
+ P += ["## Group Differences", t["groups"], ""]
576
+ if "group" in kf:
577
+ P += [one_liner(kf["group"]), ""]
578
+
579
+ P += ["## Correlations", t["corr"], ""]
580
+ if "corr" in kf:
581
+ P += [one_liner(kf["corr"]), ""]
582
+
583
+ if "redundant" in t:
584
+ P += ["## Redundant Columns", t["redundant"], ""]
585
+
586
+ P += ["## Numeric Summary", t["describe"], ""]
587
+
588
+ take = generate("Rewrite each fact as one markdown bullet. Add nothing.",
589
+ "\n".join(f"- {v}" for v in kf.values()), max_new_tokens=200)
590
+ P += ["## Takeaways",
591
+ *[l.strip() for l in take.splitlines() if l.strip().startswith(("-", "*"))][:3],
592
+ ""]
593
+
594
+ P += ["## Figures", ""]
595
+ for p in res["plots"]:
596
+ P += [f"**{os.path.basename(p)[:-4].replace('_', ' ')}**", "", f"![]({p})", ""]
597
+
598
+ report = "\n".join(P)
599
+ allowed = "\n".join([*t.values(), *kf.values(), schema_only(df, name)])
600
+ return report, audit_numbers(report, allowed)
601
+
602
+
603
  # --------------------------------------------------------------------------- #
604
+ # 7. Pipeline + Gradio plumbing
605
  # --------------------------------------------------------------------------- #
606
  DATASET_RE = re.compile(r"^[\w.-]+(/[\w.-]+)?$")
607
+ BLANK = (None, "", "", [], "", "")
608
 
609
 
610
+ def _pipeline(instruction: str, dataset: str):
611
+ """Streams: status, pdf, code, stdout, gallery, report, flags."""
 
612
  instruction = (instruction or "").strip()
613
  dataset = (dataset or "").strip()
614
  if not DATASET_RE.match(dataset):
615
+ yield ("**Invalid dataset id.** Use the form `owner/name`.", *BLANK)
616
  return
617
 
618
  shutil.rmtree(PLOTS_DIR, ignore_errors=True)
619
  os.makedirs(PLOTS_DIR, exist_ok=True)
620
 
621
+ yield (f"Loading `{dataset}` …", *BLANK)
622
  try:
623
  df = normalize(load_hf_dataframe(dataset))
624
  except Exception as e:
625
+ yield (f"**Could not load `{dataset}`.** {type(e).__name__}: {e}", *BLANK)
626
  return
627
 
628
  df, dropped = drop_index_cols(df)
629
  context = build_context(df, dataset)
630
+ note = f" Dropped index-like columns: {', '.join(dropped)}." if dropped else ""
631
+ yield (f"Loaded **{df.shape[0]} x {df.shape[1]}**.{note} Generating analysis code …",
632
+ *BLANK)
633
 
634
  code = sanitize(extract_code(
635
  generate(CODE_SYSTEM, f"{context}\n\nTask: {instruction}", max_new_tokens=1200)))
636
  res = run_code(code, df)
637
  yield (f"Attempt 1: {'ok' if res['ok'] else 'failed'}. Executing …",
638
+ None, code, res["stdout"], res["plots"], "", "")
639
 
640
  if not res["ok"]:
641
  yield ("Attempt 1 failed β€” repairing from the traceback …",
642
+ None, code, res["stdout"] + "\n" + format_error(res["error"], code),
643
  res["plots"], "", "")
644
  fix = (f"{context}\n\nThis code failed:\n```python\n{code}\n```\n\n"
645
  f"Error:\n{format_error(res['error'], code)}\n\nReturn the corrected script.")
 
647
  res = run_code(code, df)
648
 
649
  plots = sorted(glob.glob(f"{PLOTS_DIR}/*.png"))
650
+ res["plots"] = plots
651
  status = ("Code ran successfully" if res["ok"]
652
  else "Code still failing after one repair β€” report built from data only")
653
+ yield (f"{status}. Writing report …", None, code, res["stdout"], plots, "", "")
654
 
655
+ report, flags = build_report(df, res, dataset, instruction, dropped)
656
+ safe = re.sub(r"[^\w.-]", "_", dataset)
657
+ try:
658
+ pdf = report_to_pdf(report, plots, path=f"eda_report_{safe}.pdf")
659
+ pdf_note = ""
660
+ except Exception as e:
661
+ pdf, pdf_note = None, f" (PDF unavailable: {type(e).__name__})"
 
 
 
 
 
 
 
 
 
 
662
 
663
+ yield (f"Done β€” {status.lower()}, {len(plots)} figures.{pdf_note}",
664
+ pdf, code, res["stdout"], plots,
665
+ embed_images(report), ", ".join(flags) if flags else "none")
 
 
666
 
 
 
 
667
 
668
+ @spaces.GPU(duration=120)
669
+ def _run_gpu(instruction: str, dataset: str):
670
+ yield from _pipeline(instruction, dataset)
671
+
672
+
673
+ _CACHE: dict = {}
674
+
675
 
676
+ def run_agent(instruction: str, dataset: str):
677
+ """Cache check happens outside @spaces.GPU, so a repeat run costs no quota.
678
+
679
+ Free ZeroGPU is 5 min/day and the requested duration is checked upfront,
680
+ so re-demoing the same dataset would otherwise exhaust the allowance.
681
+ """
682
+ key = ((instruction or "").strip(), (dataset or "").strip())
683
+ if key in _CACHE:
684
+ status, *rest = _CACHE[key]
685
+ yield (status + " _(cached β€” no GPU used)_", *rest)
686
+ return
687
 
688
+ last = None
689
+ for out in _run_gpu(*key):
690
+ last = out
691
+ yield out
692
+ if last and last[5]: # only cache runs that produced a report
693
+ _CACHE[key] = last
694
 
695
 
696
+ # --------------------------------------------------------------------------- #
697
+ # 8. Quick Starters β€” pre-generated, instant, no model call
698
+ # --------------------------------------------------------------------------- #
699
  DEFAULT_INSTRUCTION = ("Run a comprehensive exploratory data analysis, highlighting "
700
  "missing values, distributions, and key feature correlations.")
701
 
702
+ QS_DIR = "quickstarts"
703
+ QUICKSTARTS = [
704
+ {"slug": "titanic",
705
+ "label": "🚒 Titanic β€” hidden missing values",
706
+ "blurb": "77% of `cabin` is missing, but the raw file hides it as `''`.",
707
+ "dataset": "mstz/titanic",
708
+ "instruction": DEFAULT_INSTRUCTION},
709
+ {"slug": "iris",
710
+ "label": "🌸 Iris β€” a clean baseline",
711
+ "blurb": "No missing values, a row-id column the agent drops, tight correlations.",
712
+ "dataset": "scikit-learn/iris",
713
+ "instruction": "Explore this dataset: missing values, distributions, correlations."},
714
+ {"slug": "housing",
715
+ "label": "🏠 Canada housing β€” skewed prices",
716
+ "blurb": "35,768 listings with a long price tail the agent replots on a log axis.",
717
+ "dataset": "imanmalhi/canada_realestate_listings",
718
+ "instruction": DEFAULT_INSTRUCTION},
719
+ ]
720
+
721
+
722
+ def load_quickstart(spec: dict):
723
+ """Read a pre-generated run from disk. Returns None if it was never built."""
724
+ d = os.path.join(QS_DIR, spec["slug"])
725
+ meta_path = os.path.join(d, "meta.json")
726
+ if not os.path.exists(meta_path):
727
+ return None
728
+ meta = json.load(open(meta_path))
729
+ report = open(os.path.join(d, "report.md")).read()
730
+ code = open(os.path.join(d, "code.py")).read()
731
+ stdout = open(os.path.join(d, "stdout.txt")).read()
732
+ plots = sorted(glob.glob(os.path.join(d, "plots", "*.png")))
733
+ pdf = os.path.join(d, "report.pdf")
734
+ return (
735
+ spec["instruction"], spec["dataset"],
736
+ f"**{spec['label']}** β€” pre-generated example, loaded instantly (no GPU used).",
737
+ pdf if os.path.exists(pdf) else None,
738
+ code, stdout, plots, embed_images(report),
739
+ ", ".join(meta.get("flags") or []) or "none",
740
+ )
741
+
742
+
743
+ def quickstart_handler(spec: dict):
744
+ """Serve from cache; fall back to a live run if the cache was not uploaded."""
745
+ def handler():
746
+ cached = load_quickstart(spec)
747
+ if cached is not None:
748
+ return cached
749
+ last = None
750
+ for out in run_agent(spec["instruction"], spec["dataset"]):
751
+ last = out
752
+ status, pdf, code, stdout, plots, report, flags = last
753
+ return (spec["instruction"], spec["dataset"], status, pdf,
754
+ code, stdout, plots, report, flags)
755
+ return handler
756
+
757
+
758
+ def reset_form():
759
+ return DEFAULT_INSTRUCTION, "mstz/titanic"
760
+
761
+
762
+ # --------------------------------------------------------------------------- #
763
+ # 9. UI
764
+ # --------------------------------------------------------------------------- #
765
+ CSS = """
766
+ #hero {text-align:center; padding: 6px 0 2px 0;}
767
+ .qs-card {border:1px solid var(--border-color-primary); border-radius:12px;
768
+ padding:10px 12px; height:100%;}
769
+ footer {visibility:hidden}
770
+ """
771
+
772
+ with gr.Blocks(title="EDA Agent", theme=gr.themes.Soft(), css=CSS) as demo:
773
+ gr.Markdown(
774
+ "<div id='hero'>\n\n"
775
+ "# πŸ“Š EDA Agent\n"
776
+ "**Point it at any Hugging Face dataset. It writes its own analysis code, "
777
+ "runs it, fixes it when it crashes, and hands back a report.**\n\n"
778
+ "</div>",
779
+ )
780
  gr.Markdown(
781
+ "Powered by `Qwen2.5-Coder-1.5B-Instruct`. The model writes **code and prose "
782
+ "only** β€” every table, figure and β€œwhich is highest” lookup is computed in "
783
+ "pandas, so the numbers in the report cannot be hallucinated. "
784
+ "A built-in check flags any figure in the text that is missing from the data."
 
785
  )
786
 
787
+ gr.Markdown("### ⚑ Quick Starters β€” one click, instant, no GPU used")
788
+ with gr.Row():
789
+ qs_buttons = []
790
+ for spec in QUICKSTARTS:
791
+ with gr.Column(elem_classes="qs-card"):
792
+ btn = gr.Button(spec["label"], variant="secondary", size="lg")
793
+ gr.Markdown(f"<small>{spec['blurb']}</small>")
794
+ qs_buttons.append((btn, spec))
795
+
796
+ gr.Markdown("### πŸ”Ž Or run it on any dataset")
797
  with gr.Row():
798
  instruction = gr.Textbox(label="Prompt instruction", value=DEFAULT_INSTRUCTION,
799
  lines=3, scale=3)
800
+ dataset = gr.Textbox(label="Hugging Face dataset id", value="mstz/titanic",
801
+ placeholder="owner/name", lines=1, scale=1)
802
+ with gr.Row():
803
+ run_btn = gr.Button("πŸš€ Run EDA Agent", variant="primary", size="lg", scale=4)
804
+ reset_btn = gr.Button("β†Ί Reset", variant="secondary", size="lg", scale=1)
805
+
806
+ gr.Markdown(
807
+ "<small>A fresh run takes about a minute on ZeroGPU. Free daily GPU quota is "
808
+ "shared per visitor β€” if you hit the limit, the Quick Starters above always "
809
+ "work.</small>"
810
+ )
811
+
812
  status = gr.Markdown()
813
+ pdf_file = gr.File(label="⬇️ Full report as PDF (text, tables and every figure)")
814
 
815
  with gr.Tabs():
816
+ with gr.Tab("πŸ“„ Report"):
817
  report_md = gr.Markdown()
818
  flags_box = gr.Textbox(label="Unverified numbers (fabrication check)",
819
  interactive=False)
820
+ with gr.Tab("πŸ–ΌοΈ Figures"):
821
+ gallery = gr.Gallery(label="Figures", columns=2, height=560)
822
+ with gr.Tab("🐍 Generated code"):
823
  code_box = gr.Code(language="python", label="Model-written analysis code")
824
+ with gr.Tab("πŸ–₯️ Execution output"):
825
  stdout_box = gr.Textbox(label="stdout / traceback", lines=18,
826
  interactive=False)
 
 
 
 
 
 
 
 
 
 
 
 
 
827
 
828
+ live_outputs = [status, pdf_file, code_box, stdout_box, gallery, report_md, flags_box]
829
+ qs_outputs = [instruction, dataset] + live_outputs
830
+
831
+ run_btn.click(run_agent, [instruction, dataset], live_outputs)
832
+ reset_btn.click(reset_form, None, [instruction, dataset])
833
+ for btn, spec in qs_buttons:
834
+ btn.click(quickstart_handler(spec), None, qs_outputs)
835
 
836
  if __name__ == "__main__":
837
  demo.queue().launch()