Update src/streamlit_app.py

src/streamlit_app.py

@@ -51,17 +51,17 @@ section[data-testid="stSidebar"] hr{border-color:#ffffff33 !important;}
 .metric-val{font-size:1.4rem;font-weight:700;color:#17172b !important;}
 .metric-lbl{font-size:.78rem;color:#6060a0 !important;margin-top:2px;}
 
-.big-stat-box{border-radius:12px;padding:20px 24px;text-align:center;margin-bottom:8px;}
-.big-stat-val{font-size:2.2rem;font-weight:800;margin-bottom:4px;}
-.big-stat-lbl{font-size:.82rem;font-weight:500;opacity:0.8;text-transform:uppercase;letter-spacing:.05em;}
-.big-stat-sub{font-size:.78rem;opacity:0.65;margin-top:4px;}
+.big-stat-box{border-radius:12px;padding:20px 24px;text-align:center;margin-bottom:8px;min-height:110px;}
+.big-stat-val{font-size:2.0rem;font-weight:800;margin-bottom:4px;line-height:1.1;}
+.big-stat-lbl{font-size:.80rem;font-weight:600;opacity:0.85;text-transform:uppercase;letter-spacing:.05em;}
+.big-stat-sub{font-size:.76rem;opacity:0.65;margin-top:6px;}
 
 .stat-ok{background:#edfaf3;border:2px solid #89d9ac;}
-.stat-ok .big-stat-val{color:#0a5c30;}
+.stat-ok .big-stat-val,.stat-ok .big-stat-lbl{color:#0a5c30 !important;}
 .stat-warn{background:#fffaeb;border:2px solid #f0cc7a;}
-.stat-warn .big-stat-val{color:#7a4f00;}
+.stat-warn .big-stat-val,.stat-warn .big-stat-lbl{color:#7a4f00 !important;}
 .stat-fail{background:#fff0ed;border:2px solid #f5a898;}
-.stat-fail .big-stat-val{color:#900000;}
+.stat-fail .big-stat-val,.stat-fail .big-stat-lbl{color:#900000 !important;}
 
 .card-mcar{background:#edfaf3;border:2px solid #89d9ac;border-radius:10px;padding:14px 18px;margin-bottom:10px; color:#1a1a2e !important;}
 .card-mar {background:#fffaeb;border:2px solid #f0cc7a;border-radius:10px;padding:14px 18px;margin-bottom:10px; color:#1a1a2e !important;}
@@ -384,8 +384,104 @@ def render_step2():
     miss_cols = [c for c in df.columns if df[c].isnull().any()]
     if not miss_cols: return st.success("🎉 No missing values!")
 
-    summary = pd.DataFrame({"Missing Count": df.isnull().sum(), "Missing %": (df.isnull().sum()/len(df)*100).round(2)})
-    st.dataframe(summary[summary["Missing Count"] > 0].sort_values("Missing %", ascending=False).style.background_gradient(cmap="YlOrRd"), use_container_width=True)
+    # ── Summary table ──
+    summary = pd.DataFrame({
+        "Missing Count": df[miss_cols].isnull().sum(),
+        "Missing %": (df[miss_cols].isnull().sum() / len(df) * 100).round(2)
+    }).sort_values("Missing %", ascending=False)
+    st.dataframe(
+        summary.style.background_gradient(cmap="YlOrRd", subset=["Missing %"]),
+        use_container_width=True
+    )
+
+    st.markdown("---")
+
+    # ── Missingness Heatmap ──
+    st.markdown("### 🗺️ Missingness Heatmap")
+    st.caption("Each dark stripe = a missing value in that row. Aligned stripes across columns = rows missing together (MAR signal).")
+
+    fig_h, ax_h = plt.subplots(figsize=(14, max(3, len(miss_cols) * 0.6)))
+    fig_h.patch.set_facecolor('#f8f8f8')
+    ax_h.set_facecolor('#f0f0f0')
+
+    miss_matrix = df[miss_cols].isnull().astype(int)
+    # Subsample rows for performance if large
+    if len(miss_matrix) > 2000:
+        miss_matrix = miss_matrix.sample(2000, random_state=42).reset_index(drop=True)
+
+    ax_h.imshow(
+        miss_matrix.T.values,
+        aspect='auto',
+        cmap=sns.color_palette(["#f0f0f0", "#17172b"], as_cmap=True),
+        interpolation='none'
+    )
+    ax_h.set_yticks(range(len(miss_cols)))
+    ax_h.set_yticklabels(miss_cols, fontsize=10)
+    ax_h.set_xlabel("Row index (sampled)" if len(df) > 2000 else "Row index", fontsize=10)
+    ax_h.set_title("Missing Value Pattern (dark = missing)", fontsize=12, fontweight='bold', pad=10)
+    ax_h.spines[['top','right','bottom','left']].set_visible(False)
+    plt.tight_layout()
+    st.pyplot(fig_h, use_container_width=True)
+    plt.close()
+
+    st.markdown("---")
+
+    # ── Missingness Correlation Heatmap ──
+    st.markdown("### 🔗 Missingness Correlation")
+    st.caption("Correlation between missing patterns of columns. Values near 1.0 = these columns tend to be missing in the same rows — strong MAR signal.")
+
+    if len(miss_cols) >= 2:
+        miss_indicator = df[miss_cols].isnull().astype(int)
+        corr_matrix = miss_indicator.corr()
+
+        fig_c, ax_c = plt.subplots(figsize=(max(6, len(miss_cols) * 1.2), max(5, len(miss_cols) * 1.0)))
+        fig_c.patch.set_facecolor('#f8f8f8')
+
+        mask = np.zeros_like(corr_matrix, dtype=bool)
+        mask[np.triu_indices_from(mask, k=1)] = True  # show lower triangle only
+
+        sns.heatmap(
+            corr_matrix,
+            mask=mask,
+            annot=True,
+            fmt=".2f",
+            cmap="RdYlGn",
+            vmin=-1, vmax=1,
+            center=0,
+            ax=ax_c,
+            square=True,
+            linewidths=0.5,
+            linecolor='white',
+            annot_kws={"size": 10, "weight": "bold"},
+            cbar_kws={"shrink": 0.8}
+        )
+        ax_c.set_title("Pairwise Missingness Correlation", fontsize=12, fontweight='bold', pad=12)
+        ax_c.tick_params(axis='x', rotation=45, labelsize=10)
+        ax_c.tick_params(axis='y', rotation=0, labelsize=10)
+        plt.tight_layout()
+        st.pyplot(fig_c, use_container_width=True)
+        plt.close()
+
+        # Interpretation callout
+        max_corr_pair = None
+        max_val = 0
+        for i in range(len(miss_cols)):
+            for j in range(i):
+                val = abs(corr_matrix.iloc[i, j])
+                if val > max_val:
+                    max_val = val
+                    max_corr_pair = (miss_cols[i], miss_cols[j], corr_matrix.iloc[i, j])
+
+        if max_corr_pair:
+            c1, c2, v = max_corr_pair
+            if v >= 0.9:
+                st.markdown(f'<div class="card-danger">🚨 <b>Very high missingness correlation ({v:.2f})</b> between <code>{c1}</code> and <code>{c2}</code> — these rows go missing together. Strong MAR signal; consider joint imputation (KNN/MICE).</div>', unsafe_allow_html=True)
+            elif v >= 0.5:
+                st.markdown(f'<div class="card-warn">⚠️ <b>Moderate missingness correlation ({v:.2f})</b> between <code>{c1}</code> and <code>{c2}</code> — partial co-occurrence of missingness detected.</div>', unsafe_allow_html=True)
+            else:
+                st.markdown(f'<div class="card-ok">✅ <b>Low missingness correlation (max {v:.2f})</b> — columns appear to be missing independently.</div>', unsafe_allow_html=True)
+    else:
+        st.info("Only one column with missing values — correlation requires at least two.")
 
 # ════════════════════════════════════════════════════════════════════
 #  STEP 3 — DIAGNOSTICS
@@ -578,35 +674,30 @@ def render_step4():
         skew_val  = feas["skewness"]["value"]
         skew_verd = feas["skewness"]["verdict"]
 
-        var_color  = "#900000" if var_verd == "fail"  else ("#7a4f00" if var_verd == "warn"  else "#0a5c30")
-        out_color  = "#900000" if out_verd == "fail"  else ("#7a4f00" if out_verd == "warn"  else "#0a5c30")
-        corr_color = "#900000" if corr_verd == "fail" else ("#7a4f00" if corr_verd == "warn" else "#0a5c30")
-        skew_color = "#900000" if skew_verd == "fail" else ("#7a4f00" if skew_verd == "warn" else "#0a5c30")
-
         m1.markdown(
             f'<div class="big-stat-box {COLORS[var_verd]}">'
-            f'<div class="big-stat-val" style="color:{var_color}">-{var_pct:.1f}%</div>'
+            f'<div class="big-stat-val">-{var_pct:.1f}%</div>'
             f'<div class="big-stat-lbl">Variance Change</div>'
             f'<div class="big-stat-sub">{ICONS[var_verd]} {"Safe" if var_verd=="ok" else "Caution" if var_verd=="warn" else "High Risk"}</div>'
             f'</div>', unsafe_allow_html=True
         )
         m2.markdown(
             f'<div class="big-stat-box {COLORS[out_verd]}">'
-            f'<div class="big-stat-val" style="color:{out_color}">+{new_out}</div>'
+            f'<div class="big-stat-val">+{new_out}</div>'
             f'<div class="big-stat-lbl">New Outliers Created</div>'
             f'<div class="big-stat-sub">{ICONS[out_verd]} Before: {feas["outliers"]["outliers_before"]} → After: {feas["outliers"]["outliers_after"]}</div>'
             f'</div>', unsafe_allow_html=True
         )
         m3.markdown(
             f'<div class="big-stat-box {COLORS[corr_verd]}">'
-            f'<div class="big-stat-val" style="color:{corr_color}">Δ{corr_max:.3f}</div>'
+            f'<div class="big-stat-val">Δ{corr_max:.3f}</div>'
             f'<div class="big-stat-lbl">Max Corr. Shift</div>'
             f'<div class="big-stat-sub">{ICONS[corr_verd]} {corr_verd.capitalize()}</div>'
             f'</div>', unsafe_allow_html=True
         )
         m4.markdown(
             f'<div class="big-stat-box {COLORS[skew_verd]}">'
-            f'<div class="big-stat-val" style="color:{skew_color}">{skew_val:.3f}</div>'
+            f'<div class="big-stat-val">{skew_val:.3f}</div>'
             f'<div class="big-stat-lbl">Skewness</div>'
             f'<div class="big-stat-sub">{ICONS[skew_verd]} {"Low" if abs(skew_val)<=1 else "Moderate" if abs(skew_val)<=3 else "High"} skew</div>'
             f'</div>', unsafe_allow_html=True
@@ -767,4 +858,4 @@ if   step == STEPS[0]: render_step1()
 elif step == STEPS[1]: render_step2()
 elif step == STEPS[2]: render_step3()
 elif step == STEPS[3]: render_step4()
-elif step == STEPS[4]: render_step5()
+elif step == STEPS[4]: render_step5()