Update app.py

app.py

@@ -9,6 +9,10 @@ from typing import Optional, List, Tuple, Dict, Any
 import pandas as pd
 import numpy as np
 import gradio as gr
+import seaborn as sns
+from scipy import stats
+import warnings
+warnings.filterwarnings('ignore')
 import plotly.express as px
 import plotly.graph_objects as go
 import matplotlib.pyplot as plt
@@ -121,57 +125,276 @@ def read_file_to_df(uploaded) -> pd.DataFrame:
         print("read_file_to_df error:", e)
         raise
 
-def basic_profile(df: pd.DataFrame) -> Dict[str, Any]:
+def comprehensive_data_profile(df: pd.DataFrame) -> Dict[str, Any]:
+    """Enhanced data profiling with comprehensive analysis"""
     if df is None or df.empty:
         return {}
+    
     profile = {}
     profile["rows"], profile["columns"] = df.shape
+    profile["memory_usage"] = df.memory_usage(deep=True).sum() / 1024**2  # MB
+    
+    # Basic info
     dtypes = df.dtypes.apply(lambda x: x.name).to_dict()
     profile["dtypes"] = dtypes
+    
+    # Null analysis
     nulls = df.isnull().sum().to_dict()
     profile["nulls"] = nulls
     profile["null_pct"] = {k: (v / len(df)) for k, v in nulls.items()}
+    
+    # Uniqueness analysis
     unique_counts = df.nunique(dropna=False).to_dict()
     profile["unique"] = unique_counts
-    profile["high_cardinality"] = [col for col, cnt in unique_counts.items()
-                                  if cnt > HIGH_CARD_THRESHOLD_COUNT or (cnt / len(df) > HIGH_CARD_THRESHOLD_RATIO)]
-    profile["describe"] = df.describe(include='all').to_dict()
-    profile["head"] = df.head(5).to_dict(orient="records")
+    profile["duplicate_rows"] = df.duplicated().sum()
+    profile["duplicate_pct"] = profile["duplicate_rows"] / len(df)
+    
+    # Column type classification
+    numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
+    categorical_cols = df.select_dtypes(include=["object", "category"]).columns.tolist()
+    datetime_cols = df.select_dtypes(include=["datetime64"]).columns.tolist()
+    
+    profile["numeric_cols"] = numeric_cols
+    profile["categorical_cols"] = categorical_cols
+    profile["datetime_cols"] = datetime_cols
+    
+    # High cardinality detection
+    high_cardinality = []
+    for col, cnt in unique_counts.items():
+        if col in categorical_cols and (cnt > HIGH_CARD_THRESHOLD_COUNT or 
+                                      (cnt / len(df) > HIGH_CARD_THRESHOLD_RATIO)):
+            high_cardinality.append(col)
+    profile["high_cardinality"] = high_cardinality
+    
+    # Quantitative statistics
+    quantitative_stats = {}
+    for col in numeric_cols:
+        col_data = df[col].dropna()
+        if len(col_data) > 0:
+            quantitative_stats[col] = {
+                "mean": col_data.mean(),
+                "median": col_data.median(),
+                "std": col_data.std(),
+                "min": col_data.min(),
+                "max": col_data.max(),
+                "q1": col_data.quantile(0.25),
+                "q3": col_data.quantile(0.75),
+                "skew": col_data.skew(),
+                "kurtosis": col_data.kurtosis(),
+                "zeros": (col_data == 0).sum(),
+                "zeros_pct": (col_data == 0).sum() / len(col_data),
+                "outliers": detect_outliers_iqr(col_data)
+            }
+    profile["quantitative_stats"] = quantitative_stats
+    
+    # Qualitative statistics
+    qualitative_stats = {}
+    for col in categorical_cols:
+        col_data = df[col].dropna()
+        if len(col_data) > 0:
+            value_counts = col_data.value_counts()
+            qualitative_stats[col] = {
+                "top_value": value_counts.index[0] if len(value_counts) > 0 else None,
+                "top_freq": value_counts.iloc[0] if len(value_counts) > 0 else 0,
+                "top_freq_pct": value_counts.iloc[0] / len(col_data) if len(value_counts) > 0 else 0,
+                "unique_values": len(value_counts),
+                "entropy": stats.entropy(value_counts.values) if len(value_counts) > 0 else 0
+            }
+    profile["qualitative_stats"] = qualitative_stats
+    
+    # Data quality indicators
+    quality_metrics = {
+        "completeness": (len(df) - df.isnull().sum().sum()) / (len(df) * len(df.columns)),
+        "uniqueness": 1 - (profile["duplicate_pct"]),
+        "validity": {}  # Could be extended with domain-specific rules
+    }
+    profile["quality_metrics"] = quality_metrics
+    
+    # Correlation matrix for numeric columns
+    if len(numeric_cols) > 1:
+        profile["correlation_matrix"] = df[numeric_cols].corr().round(3).to_dict()
+    else:
+        profile["correlation_matrix"] = {}
+    
+    # Sample data
+    profile["head"] = df.head(10).to_dict(orient="records")
+    
     return profile
 
-def profile_to_markdown(profile: Dict[str, Any]) -> str:
+def detect_outliers_iqr(series):
+    """Detect outliers using IQR method"""
+    Q1 = series.quantile(0.25)
+    Q3 = series.quantile(0.75)
+    IQR = Q3 - Q1
+    lower_bound = Q1 - 1.5 * IQR
+    upper_bound = Q3 + 1.5 * IQR
+    return ((series < lower_bound) | (series > upper_bound)).sum()
+
+def profile_to_enhanced_markdown(profile: Dict[str, Any]) -> str:
+    """Convert comprehensive profile to rich markdown with visual indicators"""
     if not profile:
         return "No data loaded."
+    
     md = []
-    md.append(f"**Rows:** {profile['rows']}  \n**Columns:** {profile['columns']}\n")
-    md.append("### Column summary (dtypes / null% / unique)\n")
-    md.append("| Column | Dtype | Nulls | Null % | Unique |\n|---:|---|---:|---:|---:|\n")
+    
+    # Header with key metrics
+    md.append(f"## 📊 Comprehensive Data Profile Report")
+    md.append("---")
+    
+    # Dataset Overview
+    md.append("### 🎯 Dataset Overview")
+    md.append(f"- **Rows:** {profile['rows']:,}")
+    md.append(f"- **Columns:** {profile['columns']}")
+    md.append(f"- **Memory Usage:** {profile['memory_usage']:.2f} MB")
+    md.append(f"- **Duplicate Rows:** {profile['duplicate_rows']} ({profile['duplicate_pct']:.2%})")
+    md.append("")
+    
+    # Data Quality Scorecard
+    md.append("### 📈 Data Quality Scorecard")
+    quality = profile["quality_metrics"]
+    md.append(f"- **Completeness:** {quality['completeness']:.2%}")
+    md.append(f"- **Uniqueness:** {quality['uniqueness']:.2%}")
+    md.append("")
+    
+    # Column Type Summary
+    md.append("### 🗂️ Column Type Summary")
+    md.append(f"- **Numeric Columns:** {len(profile['numeric_cols'])}")
+    md.append(f"- **Categorical Columns:** {len(profile['categorical_cols'])}")
+    md.append(f"- **DateTime Columns:** {len(profile['datetime_cols'])}")
+    md.append("")
+    
+    # Detailed Column Analysis
+    md.append("### 🔍 Detailed Column Analysis")
+    md.append("| Column | Type | Nulls | Null % | Unique | Completeness | Issues |")
+    md.append("|--------|------|-------|---------|--------|--------------|--------|")
+    
     for col in profile["dtypes"].keys():
         dtype = profile["dtypes"][col]
         nulls = profile["nulls"].get(col, 0)
-        pct = f"{profile['null_pct'].get(col,0):.2%}"
+        null_pct = profile["null_pct"].get(col, 0)
         uniq = profile["unique"].get(col, 0)
-        md.append(f"| {col} | {dtype} | {nulls} | {pct} | {uniq} |\n")
+        completeness = 1 - null_pct
+        
+        # Issue indicators
+        issues = []
+        if null_pct > 0.5:
+            issues.append("🔴 High nulls")
+        elif null_pct > 0.2:
+            issues.append("🟡 Medium nulls")
+        
+        if col in profile["high_cardinality"]:
+            issues.append("🔵 High cardinality")
+            
+        if col in profile["numeric_cols"]:
+            stats = profile["quantitative_stats"].get(col, {})
+            outliers = stats.get("outliers", 0)
+            if outliers > 0:
+                issues.append("⚫ Outliers")
+        
+        issues_str = ", ".join(issues) if issues else "✅ Good"
+        
+        md.append(f"| {col} | {dtype} | {nulls} | {null_pct:.2%} | {uniq} | {completeness:.2%} | {issues_str} |")
+    md.append("")
+    
+    # Quantitative Columns Deep Dive
+    if profile["quantitative_stats"]:
+        md.append("### 📈 Quantitative Columns Analysis")
+        md.append("| Column | Mean | Std | Min | Max | Skew | Outliers | Zeros |")
+        md.append("|--------|------|-----|-----|-----|------|----------|-------|")
+        
+        for col, stats in profile["quantitative_stats"].items():
+            md.append(f"| {col} | {stats['mean']:.2f} | {stats['std']:.2f} | {stats['min']:.2f} | {stats['max']:.2f} | {stats['skew']:.2f} | {stats['outliers']} | {stats['zeros']} |")
+        md.append("")
+    
+    # Qualitative Columns Deep Dive
+    if profile["qualitative_stats"]:
+        md.append("### 📊 Qualitative Columns Analysis")
+        md.append("| Column | Top Value | Top Freq | Top % | Unique | Entropy |")
+        md.append("|--------|-----------|----------|-------|--------|---------|")
+        
+        for col, stats in profile["qualitative_stats"].items():
+            top_value = str(stats["top_value"])[:20] + "..." if len(str(stats["top_value"])) > 20 else str(stats["top_value"])
+            md.append(f"| {col} | {top_value} | {stats['top_freq']} | {stats['top_freq_pct']:.2%} | {stats['unique_values']} | {stats['entropy']:.2f} |")
+        md.append("")
+    
+    # High Cardinality Warning
     if profile["high_cardinality"]:
-        md.append("\n**High cardinality columns (auto-detected):** " + ", ".join(profile["high_cardinality"]) + "\n")
-        md.append("\n### Sample rows\n")
-    # safe markdown conversion of sample rows without requiring 'tabulate'
+        md.append("### ⚠️ High Cardinality Columns")
+        md.append("The following columns have high cardinality (may impact modeling):")
+        for col in profile["high_cardinality"]:
+            md.append(f"- **{col}**: {profile['unique'][col]} unique values")
+        md.append("")
+    
+    # Correlation Highlights (if available)
+    if profile["correlation_matrix"]:
+        md.append("### 🔗 Correlation Highlights")
+        corr_matrix = profile["correlation_matrix"]
+        numeric_cols = list(corr_matrix.keys())
+        
+        # Find strong correlations
+        strong_corrs = []
+        for i, col1 in enumerate(numeric_cols):
+            for j, col2 in enumerate(numeric_cols):
+                if i < j:  # Avoid duplicates and self-correlation
+                    corr = abs(corr_matrix[col1][col2])
+                    if corr > 0.7:
+                        strong_corrs.append((col1, col2, corr_matrix[col1][col2]))
+        
+        if strong_corrs:
+            md.append("**Strong Correlations (|r| > 0.7):**")
+            for col1, col2, corr in sorted(strong_corrs, key=lambda x: abs(x[2]), reverse=True):
+                md.append(f"- {col1} ↔ {col2}: {corr:.3f}")
+        else:
+            md.append("No strong correlations found among numeric columns.")
+        md.append("")
+    
+    return "\n".join(md)
+
+def create_distribution_plots(df: pd.DataFrame, profile: Dict[str, Any]):
+    """Create distribution plots for numeric and categorical columns"""
+    numeric_cols = profile.get("numeric_cols", [])
+    categorical_cols = profile.get("categorical_cols", [])
+    
+    plots = {}
+    
+    # Numeric distributions
+    for col in numeric_cols[:4]:  # Limit to first 4 for performance
         try:
-            df_head = pd.DataFrame(profile.get("head", []))
-            if df_head.empty:
-                md.append("No sample rows available.\n")
-            else:
-                headers = list(df_head.columns)
-                md.append("| " + " | ".join(headers) + " |\n")
-                md.append("|" + "|".join(["---"] * len(headers)) + "|\n")
-            # rows
-                for _, row in df_head.iterrows():
-                    row_vals = [str(row.get(h, "")) for h in headers]
-                    md.append("| " + " | ".join(row_vals) + " |\n")
-        except Exception:
-            md.append(str(profile.get("head", "")) + "\n")
-        return "\n".join(md)
+            fig = px.histogram(df, x=col, title=f"Distribution of {col}", 
+                             marginal="box", nbins=50)
+            plots[f"num_{col}"] = fig
+        except Exception as e:
+            print(f"Plot error for {col}: {e}")
+    
+    # Categorical distributions (top 3)
+    for col in categorical_cols[:3]:
+        try:
+            value_counts = df[col].value_counts().head(10)
+            fig = px.bar(x=value_counts.index, y=value_counts.values,
+                        title=f"Top 10 Values in {col}")
+            fig.update_layout(xaxis_title=col, yaxis_title="Count")
+            plots[f"cat_{col}"] = fig
+        except Exception as e:
+            print(f"Plot error for {col}: {e}")
+    
+    return plots
 
+def create_correlation_plot(df: pd.DataFrame, profile: Dict[str, Any]):
+    """Create correlation heatmap"""
+    numeric_cols = profile.get("numeric_cols", [])
+    if len(numeric_cols) < 2:
+        return None
+    
+    try:
+        corr_matrix = df[numeric_cols].corr()
+        fig = px.imshow(corr_matrix, 
+                       title="Correlation Matrix",
+                       color_continuous_scale="RdBu_r",
+                       aspect="auto")
+        return fig
+    except Exception as e:
+        print(f"Correlation plot error: {e}")
+        return None
 # -----------------------------
 # Data cleaning & feature engineering helpers
 # -----------------------------
@@ -602,13 +825,27 @@ with gr.Blocks(css=css, title="DataSynth — Analytics Hub") as demo:
         gr.Markdown("## Workspace")
         with gr.Tabs():
             with gr.TabItem("Data"):
-                gr.Markdown("Upload CSV / Excel / JSON for profiling and cleaning.")
+                gr.Markdown("Upload CSV / Excel / JSON for comprehensive profiling and analysis.")
                 upload = gr.File(label="Upload CSV / Excel / JSON", file_types=[".csv", ".xlsx", ".xls", ".json"])
                 with gr.Row():
-                    load_btn = gr.Button("Load & Profile")
-                    download_raw_btn = gr.Button("Download Raw CSV")
-                profile_md = gr.Markdown("No dataset loaded.")
-                sample_table = gr.Dataframe(interactive=False)
+                    profile_btn = gr.Button("📊 Data Profiling", variant="primary")
+                    view_data_btn = gr.Button("👀 View Data", variant="secondary")
+                    download_raw_btn = gr.Button("📥 Download Raw CSV")
+                with gr.Tabs() as data_tabs:
+                    with gr.TabItem("Profile Report"):
+                        profile_md = gr.Markdown("No dataset loaded. Upload data and click 'Data Profiling'.")
+
+                    with gr.TabItem("Data Preview"):
+                        sample_table = gr.Dataframe(interactive=False, label="Sample Data (First 100 rows)")
+
+                    with gr.TabItem("Visualizations"):
+                        with gr.Row():
+                            dist_plot1 = gr.Plot(label="Distribution Plot 1")
+                            dist_plot2 = gr.Plot(label="Distribution Plot 2")
+                        with gr.Row():
+                            dist_plot3 = gr.Plot(label="Distribution Plot 3")
+                            corr_plot = gr.Plot(label="Correlation Heatmap")
+                            
             with gr.TabItem("Prepare"):
                 gr.Markdown("Data cleaning and feature engineering options.")
                 with gr.Row():
@@ -666,21 +903,50 @@ with gr.Blocks(css=css, title="DataSynth — Analytics Hub") as demo:
     def _load_and_profile(uploaded):
         try:
             if uploaded is None:
-                return gr.update(value="No file uploaded."), pd.DataFrame(), None, None
+                return (gr.update(value="No file uploaded."),
+                        pd.DataFrame(), None, None,
+                        None, None, None, None)
             df = read_file_to_df(uploaded)
-            prof = basic_profile(df)
-            md = profile_to_markdown(prof)
+            prof = comprehensive_data_profile(df)
+            md = profile_to_enhanced_markdown(prof)
+            #create visulaizations
+            plots = create_distribution_plots(df, prof)
+            corr_plot = create_correlation_plot(df, prof)
+
+            # Prepare plot outputs
+            plot1 = plots.get(list(plots.keys())[0]) if plots else None
+            plot2 = plots.get(list(plots.keys())[1]) if len(plots) > 1 else None
+            plot3 = plots.get(list(plots.keys())[2]) if len(plots) > 2 else None
             # prepare choices for date and target selectors
-            cols = df.columns.tolist()
+            #cols = df.columns.tolist()
             # save df to temp csv for persistence (store path in state)
             tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".csv")
             df.to_csv(tmp.name, index=False)
-            return md, df.head(100), tmp.name, prof
+            
+            return (md, df.head(100), tmp.name, prof,
+                    plot1, plot2, plot3, corr_plot)
         except Exception as e:
-            return gr.update(value=f"Error loading file: {e}"), pd.DataFrame(), None, None
+            return (gr.update(value=f"Error loading file: {e}"),
+                    pd.DataFrame(), None, None,
+                    None, None, None, None)
 
-    load_btn.click(fn=_load_and_profile, inputs=[upload], outputs=[profile_md, sample_table, df_state, profile_state])
+    profile_btn.click(fn=_load_and_profile,
+                      inputs=[upload],
+                      outputs=[profile_md, sample_table, df_state, profile_state,
+                               dist_plot1, dist_plot2, dist_plot3, corr_plot])
+
+    def _view_data(uploaded):
+        try:
+            if uploaded is None:
+                return pd.DataFrame()
+            df = read_file_to_df(uploaded)
+            return df.head(100)
+        except Exception as e:
+            return pd.DataFrame()
 
+    view_data_btn.click(fn=_view_data, inputs=[upload], outputs=[sample_table])
+            
+        
     # download raw
     def _download_raw(df_path):
         if not df_path: