app.py

import gradio as gr
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

def clean_and_analyze(file):
    if file is None:
        return "Please upload a CSV file", None
    
    # Load raw data
    df_raw = pd.read_csv(file.name)
    
    # ========== PART 1: SHOW MISSING VALUES BEFORE CLEANING ==========
    report = ""
    report += "="*60 + "\n"
    report += "STEP 1: RAW DATA - MISSING VALUES DETECTION\n"
    report += "="*60 + "\n\n"
    
    missing_before = df_raw.isnull().sum()
    missing_pct_before = (missing_before / len(df_raw) * 100).round(2)
    
    if missing_before.sum() == 0:
        report += "No missing values found in raw data.\n\n"
    else:
        report += "Columns with missing values BEFORE cleaning:\n"
        for col in df_raw.columns:
            if missing_before[col] > 0:
                report += f"  - {col}: {missing_before[col]} missing ({missing_pct_before[col]}%)\n"
        report += f"\nTotal missing values: {missing_before.sum()}\n\n"
    
    # ========== PART 2: PERFORM CLEANING ==========
    report += "="*60 + "\n"
    report += "STEP 2: DATA CLEANING PROCESS\n"
    report += "="*60 + "\n\n"
    
    df_clean = df_raw.copy()
    cleaning_actions = []
    
    # 2.1 Remove columns that are 100% empty
    empty_cols = [col for col in df_clean.columns if df_clean[col].isnull().all()]
    if len(empty_cols) > 0:
        df_clean = df_clean.drop(columns=empty_cols)
        cleaning_actions.append(f"Removed {len(empty_cols)} completely empty columns: {empty_cols}")
    
    # 2.2 Remove duplicate rows
    before_rows = len(df_clean)
    df_clean = df_clean.drop_duplicates()
    dup_removed = before_rows - len(df_clean)
    if dup_removed > 0:
        cleaning_actions.append(f"Removed {dup_removed} duplicate rows")
    
    # 2.3 Fill numeric columns with median
    numeric_cols = df_clean.select_dtypes(include=[np.number]).columns
    for col in numeric_cols:
        if df_clean[col].isnull().sum() > 0:
            median_val = df_clean[col].median()
            df_clean[col] = df_clean[col].fillna(median_val)
            cleaning_actions.append(f"Filled {col} with median value: {median_val:.2f}")
    
    # 2.4 Fill categorical columns with mode
    cat_cols = df_clean.select_dtypes(include=['object']).columns
    for col in cat_cols:
        if df_clean[col].isnull().sum() > 0:
            mode_val = df_clean[col].mode()[0] if len(df_clean[col].mode()) > 0 else "Unknown"
            df_clean[col] = df_clean[col].fillna(mode_val)
            cleaning_actions.append(f"Filled {col} with most common value: {mode_val}")
    
    # Log cleaning actions
    if len(cleaning_actions) == 0:
        report += "No cleaning was needed. Data was already clean.\n\n"
    else:
        for action in cleaning_actions:
            report += f"  - {action}\n"
        report += "\n"
    
    # ========== PART 3: SHOW MISSING VALUES AFTER CLEANING ==========
    report += "="*60 + "\n"
    report += "STEP 3: VERIFICATION - MISSING VALUES AFTER CLEANING\n"
    report += "="*60 + "\n\n"
    
    missing_after = df_clean.isnull().sum()
    if missing_after.sum() == 0:
        report += "SUCCESS: No missing values remain. Data is fully clean.\n\n"
    else:
        report += "Warning: Some missing values still exist:\n"
        for col in df_clean.columns:
            if missing_after[col] > 0:
                report += f"  - {col}: {missing_after[col]} missing\n"
        report += "\n"
    
    # ========== PART 4: DATASET OVERVIEW AFTER CLEANING ==========
    report += "="*60 + "\n"
    report += "STEP 4: CLEANED DATASET OVERVIEW\n"
    report += "="*60 + "\n\n"
    
    report += f"Original shape: {df_raw.shape[0]} rows, {df_raw.shape[1]} columns\n"
    report += f"Cleaned shape: {df_clean.shape[0]} rows, {df_clean.shape[1]} columns\n\n"
    
    report += "Columns in cleaned dataset:\n"
    for col in df_clean.columns:
        report += f"  - {col}: {df_clean[col].dtype}\n"
    report += "\n"
    
    # ========== PART 5: BASIC STATISTICS ON CLEANED DATA ==========
    numeric_cols_clean = df_clean.select_dtypes(include=[np.number]).columns
    report += "="*60 + "\n"
    report += "STEP 5: BASIC STATISTICS (On Cleaned Data)\n"
    report += "="*60 + "\n\n"
    
    if len(numeric_cols_clean) > 0:
        for col in numeric_cols_clean[:5]:
            report += f"{col}:\n"
            report += f"  Mean:   {df_clean[col].mean():.2f}\n"
            report += f"  Median: {df_clean[col].median():.2f}\n"
            report += f"  Min:    {df_clean[col].min():.2f}\n"
            report += f"  Max:    {df_clean[col].max():.2f}\n"
            report += f"  Std:    {df_clean[col].std():.2f}\n\n"
    else:
        report += "No numeric columns found.\n\n"
    
    # ========== PART 6: KEY INSIGHTS ==========
    report += "="*60 + "\n"
    report += "STEP 6: KEY INSIGHTS\n"
    report += "="*60 + "\n\n"
    
    report += f"- Cleaned dataset has {df_clean.shape[0]} rows and {df_clean.shape[1]} columns\n"
    report += f"- {len(numeric_cols_clean)} numeric columns available for analysis\n"
    report += f"- {df_clean.shape[1] - len(numeric_cols_clean)} categorical columns\n"
    
    if missing_before.sum() > 0:
        report += f"- Cleaned {missing_before.sum()} missing values\n"
    
    if len(numeric_cols_clean) >= 2:
        corr_matrix = df_clean[numeric_cols_clean].corr()
        max_corr = 0
        max_pair = ""
        for i in range(len(corr_matrix.columns)):
            for j in range(i+1, len(corr_matrix.columns)):
                corr_val = abs(corr_matrix.iloc[i,j])
                if corr_val > max_corr:
                    max_corr = corr_val
                    max_pair = f"{corr_matrix.columns[i]} and {corr_matrix.columns[j]}"
        if max_corr > 0:
            report += f"- Strongest correlation: {max_pair} ({max_corr:.2f})\n"
    
    # ========== PART 7: VISUALIZATIONS ==========
    fig = plt.figure(figsize=(14, 12))
    
    # Plot 1: Histogram of first numeric column
    if len(numeric_cols_clean) > 0:
        plt.subplot(2, 2, 1)
        plt.hist(df_clean[numeric_cols_clean[0]].dropna(), bins=30, edgecolor='black')
        plt.xlabel(numeric_cols_clean[0])
        plt.ylabel('Frequency')
        plt.title(f'Distribution of {numeric_cols_clean[0]} (After Cleaning)')
        plt.grid(True, alpha=0.3)
    
    # Plot 2: Boxplot
    if len(numeric_cols_clean) > 0:
        plt.subplot(2, 2, 2)
        col_for_box = numeric_cols_clean[1] if len(numeric_cols_clean) > 1 else numeric_cols_clean[0]
        plt.boxplot(df_clean[col_for_box].dropna())
        plt.ylabel(col_for_box)
        plt.title(f'Boxplot of {col_for_box} (After Cleaning)')
        plt.grid(True, alpha=0.3)
    
    # Plot 3: Correlation Heatmap
    if len(numeric_cols_clean) >= 2:
        plt.subplot(2, 2, 3)
        corr = df_clean[numeric_cols_clean].corr()
        sns.heatmap(corr, annot=True, cmap='coolwarm', center=0)
        plt.title('Correlation Matrix (After Cleaning)')
    
    # Plot 4: Bar chart or trend
    plt.subplot(2, 2, 4)
    cat_cols_clean = df_clean.select_dtypes(include=['object']).columns
    if len(cat_cols_clean) > 0:
        counts = df_clean[cat_cols_clean[0]].value_counts().head(10)
        plt.bar(range(len(counts)), counts.values)
        plt.xticks(range(len(counts)), counts.index, rotation=45, ha='right')
        plt.ylabel('Count')
        plt.title(f'Top values in {cat_cols_clean[0]}')
    elif len(numeric_cols_clean) >= 2:
        plt.plot(df_clean[numeric_cols_clean[0]].head(50).values, marker='o')
        plt.xlabel('Row Index')
        plt.ylabel(numeric_cols_clean[0])
        plt.title(f'Trend of {numeric_cols_clean[0]} (First 50 rows)')
    
    plt.tight_layout()
    
    return report, fig

# Create the interface
demo = gr.Interface(
    fn=clean_and_analyze,
    inputs=gr.File(label="Upload CSV File", file_types=[".csv"]),
    outputs=[
        gr.Textbox(label="Complete Analysis Report", lines=35),
        gr.Plot(label="Visualizations")
    ],
    title="Data Analysis Agent - With Automatic Data Cleaning",
    description="Upload any CSV file. The agent will: 1) Show missing values, 2) Clean/fill missing data, 3) Show analysis on cleaned data, 4) Generate visualizations."
)

demo.launch()