Update app.py

app.py

@@ -225,8 +225,12 @@ def run_code(code: str, df: pd.DataFrame) -> dict:
     except Exception:
         err = traceback.format_exc(limit=3)
 
-    for i, num in enumerate(plt.get_fignums()):             # rescue orphan figures
-        plt.figure(num).savefig(f"{PLOTS_DIR}/rescued_{i}.png", bbox_inches="tight")
+    # safety net: model called plt.show(), or crashed with a figure still open
+    for i, num in enumerate(plt.get_fignums(), start=1):
+        fig = plt.figure(num)
+        if not fig.get_axes():                 # skip blank figures
+            continue
+        fig.savefig(f"{PLOTS_DIR}/figure_{i}.png", bbox_inches="tight")
     plt.close("all")
 
     return {"ok": err is None, "stdout": buf.getvalue(), "error": err,
@@ -247,12 +251,40 @@ def format_error(err: str, code: str) -> str:
 # --------------------------------------------------------------------------- #
 # 5. Deterministic facts and tables
 # --------------------------------------------------------------------------- #
+REDUNDANT_R = 0.99      # |r| at or above this is a duplicate encoding, not a finding
+
+
+def corr_pairs(df: pd.DataFrame):
+    """Split correlations into real findings and near-duplicate columns."""
+    num = df.select_dtypes(include="number")
+    if len(num.columns) < 2:
+        return [], []
+    corr = num.corr()
+    pr = corr.where(~np.eye(len(corr), dtype=bool)).stack().sort_values(
+        key=abs, ascending=False)
+    seen, dup, real = set(), [], []
+    for (a, b), v in pr.items():
+        if (b, a) in seen:
+            continue
+        seen.add((a, b))
+        row = {"pair": f"{a} ~ {b}", "r": round(v, 3)}
+        (dup if abs(v) >= REDUNDANT_R else real).append(row)
+    return real, dup
+
+
 def make_tables(df: pd.DataFrame, max_card: int = 6) -> dict:
     t = {}
     n = df.isna().sum()
     n = n[n > 0].sort_values(ascending=False)
-    t["missing"] = (pd.DataFrame({"missing": n, "pct": (100 * n / len(df)).round(1)})
-                    .to_markdown() if len(n) else "_No missing values._")
+    if len(n):
+        head = n.head(15)
+        tbl = pd.DataFrame({"missing": head,
+                            "pct": (100 * head / len(df)).round(1)}).to_markdown()
+        if len(n) > 15:
+            tbl += f"\n\n_+{len(n) - 15} more columns with missing values._"
+        t["missing"] = tbl
+    else:
+        t["missing"] = "_No missing values._"
 
     bins = [c for c in df.select_dtypes(include="number") if df[c].nunique() == 2]
     blocks = [
@@ -263,21 +295,18 @@ def make_tables(df: pd.DataFrame, max_card: int = 6) -> dict:
     ]
     t["groups"] = "\n\n".join(blocks) if blocks else "_No low-cardinality groupings._"
 
+    real, dup = corr_pairs(df)
+    t["corr"] = (pd.DataFrame(real[:8]).to_markdown(index=False)
+                 if real else "_Too few numeric columns._")
+    if dup:
+        t["redundant"] = (
+            f"_{len(dup)} column pairs correlate at |r| >= {REDUNDANT_R} — likely duplicate "
+            "encodings rather than findings._\n\n"
+            + pd.DataFrame(dup[:8]).to_markdown(index=False))
+
     num = df.select_dtypes(include="number")
-    corr = num.corr()
-    pr = corr.where(~np.eye(len(corr), dtype=bool)).stack().sort_values(
-        key=abs, ascending=False)
-    seen, rows = set(), []
-    for (a, b), v in pr.items():
-        if (b, a) in seen:
-            continue
-        seen.add((a, b))
-        rows.append({"pair": f"{a} ~ {b}", "r": round(v, 3)})
-        if len(rows) >= 8:
-            break
-    t["corr"] = (pd.DataFrame(rows).to_markdown(index=False)
-                 if rows else "_Too few numeric columns._")
-    t["describe"] = num.describe().round(2).to_markdown() if len(num.columns) else "_None._"
+    t["describe"] = (num.describe().T.round(2).to_markdown()      # transposed
+                     if len(num.columns) else "_None._")
     return t
 
 
@@ -304,14 +333,11 @@ def key_facts(df: pd.DataFrame, max_card: int = 6) -> dict:
         b, c, k, v = best
         f["group"] = f"the highest mean {b} is {v:.3f}, for {c} = {k}"
 
-    num = df.select_dtypes(include="number")
-    if len(num.columns) >= 2:
-        corr = num.corr()
-        pr = corr.where(~np.eye(len(corr), dtype=bool)).stack().sort_values(
-            key=abs, ascending=False)
-        (a, b2), v = pr.index[0], pr.iloc[0]
-        f["corr"] = (f"the strongest correlation is {a} ~ {b2} at r = {v:+.3f} "
-                     f"({'positive' if v > 0 else 'negative'})")
+    real, _ = corr_pairs(df)
+    if real:
+        r = real[0]
+        f["corr"] = (f"the strongest correlation is {r['pair']} at r = {r['r']:+.3f} "
+                     f"({'positive' if r['r'] > 0 else 'negative'})")
     return f
 
 
@@ -415,6 +441,8 @@ def run_agent(instruction: str, dataset: str):
     P += ["## Correlations", t["corr"], ""]
     if "corr" in kf:
         P += [one_liner(kf["corr"]), ""]
+    if "redundant" in t:
+        P += ["## Redundant Columns", t["redundant"], ""]
     P += ["## Numeric Summary", t["describe"], ""]
 
     take = generate("Rewrite each fact as one markdown bullet. Add nothing.",