import gradio as gr import pandas as pd import numpy as np import matplotlib.pyplot as plt import seaborn as sns def clean_and_analyze(file): if file is None: return "Please upload a CSV file", None # Load raw data df_raw = pd.read_csv(file.name) # ========== PART 1: SHOW MISSING VALUES BEFORE CLEANING ========== report = "" report += "="*60 + "\n" report += "STEP 1: RAW DATA - MISSING VALUES DETECTION\n" report += "="*60 + "\n\n" missing_before = df_raw.isnull().sum() missing_pct_before = (missing_before / len(df_raw) * 100).round(2) if missing_before.sum() == 0: report += "No missing values found in raw data.\n\n" else: report += "Columns with missing values BEFORE cleaning:\n" for col in df_raw.columns: if missing_before[col] > 0: report += f" - {col}: {missing_before[col]} missing ({missing_pct_before[col]}%)\n" report += f"\nTotal missing values: {missing_before.sum()}\n\n" # ========== PART 2: PERFORM CLEANING ========== report += "="*60 + "\n" report += "STEP 2: DATA CLEANING PROCESS\n" report += "="*60 + "\n\n" df_clean = df_raw.copy() cleaning_actions = [] # 2.1 Remove columns that are 100% empty empty_cols = [col for col in df_clean.columns if df_clean[col].isnull().all()] if len(empty_cols) > 0: df_clean = df_clean.drop(columns=empty_cols) cleaning_actions.append(f"Removed {len(empty_cols)} completely empty columns: {empty_cols}") # 2.2 Remove duplicate rows before_rows = len(df_clean) df_clean = df_clean.drop_duplicates() dup_removed = before_rows - len(df_clean) if dup_removed > 0: cleaning_actions.append(f"Removed {dup_removed} duplicate rows") # 2.3 Fill numeric columns with median numeric_cols = df_clean.select_dtypes(include=[np.number]).columns for col in numeric_cols: if df_clean[col].isnull().sum() > 0: median_val = df_clean[col].median() df_clean[col] = df_clean[col].fillna(median_val) cleaning_actions.append(f"Filled {col} with median value: {median_val:.2f}") # 2.4 Fill categorical columns with mode cat_cols = df_clean.select_dtypes(include=['object']).columns for col in cat_cols: if df_clean[col].isnull().sum() > 0: mode_val = df_clean[col].mode()[0] if len(df_clean[col].mode()) > 0 else "Unknown" df_clean[col] = df_clean[col].fillna(mode_val) cleaning_actions.append(f"Filled {col} with most common value: {mode_val}") # Log cleaning actions if len(cleaning_actions) == 0: report += "No cleaning was needed. Data was already clean.\n\n" else: for action in cleaning_actions: report += f" - {action}\n" report += "\n" # ========== PART 3: SHOW MISSING VALUES AFTER CLEANING ========== report += "="*60 + "\n" report += "STEP 3: VERIFICATION - MISSING VALUES AFTER CLEANING\n" report += "="*60 + "\n\n" missing_after = df_clean.isnull().sum() if missing_after.sum() == 0: report += "SUCCESS: No missing values remain. Data is fully clean.\n\n" else: report += "Warning: Some missing values still exist:\n" for col in df_clean.columns: if missing_after[col] > 0: report += f" - {col}: {missing_after[col]} missing\n" report += "\n" # ========== PART 4: DATASET OVERVIEW AFTER CLEANING ========== report += "="*60 + "\n" report += "STEP 4: CLEANED DATASET OVERVIEW\n" report += "="*60 + "\n\n" report += f"Original shape: {df_raw.shape[0]} rows, {df_raw.shape[1]} columns\n" report += f"Cleaned shape: {df_clean.shape[0]} rows, {df_clean.shape[1]} columns\n\n" report += "Columns in cleaned dataset:\n" for col in df_clean.columns: report += f" - {col}: {df_clean[col].dtype}\n" report += "\n" # ========== PART 5: BASIC STATISTICS ON CLEANED DATA ========== numeric_cols_clean = df_clean.select_dtypes(include=[np.number]).columns report += "="*60 + "\n" report += "STEP 5: BASIC STATISTICS (On Cleaned Data)\n" report += "="*60 + "\n\n" if len(numeric_cols_clean) > 0: for col in numeric_cols_clean[:5]: report += f"{col}:\n" report += f" Mean: {df_clean[col].mean():.2f}\n" report += f" Median: {df_clean[col].median():.2f}\n" report += f" Min: {df_clean[col].min():.2f}\n" report += f" Max: {df_clean[col].max():.2f}\n" report += f" Std: {df_clean[col].std():.2f}\n\n" else: report += "No numeric columns found.\n\n" # ========== PART 6: KEY INSIGHTS ========== report += "="*60 + "\n" report += "STEP 6: KEY INSIGHTS\n" report += "="*60 + "\n\n" report += f"- Cleaned dataset has {df_clean.shape[0]} rows and {df_clean.shape[1]} columns\n" report += f"- {len(numeric_cols_clean)} numeric columns available for analysis\n" report += f"- {df_clean.shape[1] - len(numeric_cols_clean)} categorical columns\n" if missing_before.sum() > 0: report += f"- Cleaned {missing_before.sum()} missing values\n" if len(numeric_cols_clean) >= 2: corr_matrix = df_clean[numeric_cols_clean].corr() max_corr = 0 max_pair = "" for i in range(len(corr_matrix.columns)): for j in range(i+1, len(corr_matrix.columns)): corr_val = abs(corr_matrix.iloc[i,j]) if corr_val > max_corr: max_corr = corr_val max_pair = f"{corr_matrix.columns[i]} and {corr_matrix.columns[j]}" if max_corr > 0: report += f"- Strongest correlation: {max_pair} ({max_corr:.2f})\n" # ========== PART 7: VISUALIZATIONS ========== fig = plt.figure(figsize=(14, 12)) # Plot 1: Histogram of first numeric column if len(numeric_cols_clean) > 0: plt.subplot(2, 2, 1) plt.hist(df_clean[numeric_cols_clean[0]].dropna(), bins=30, edgecolor='black') plt.xlabel(numeric_cols_clean[0]) plt.ylabel('Frequency') plt.title(f'Distribution of {numeric_cols_clean[0]} (After Cleaning)') plt.grid(True, alpha=0.3) # Plot 2: Boxplot if len(numeric_cols_clean) > 0: plt.subplot(2, 2, 2) col_for_box = numeric_cols_clean[1] if len(numeric_cols_clean) > 1 else numeric_cols_clean[0] plt.boxplot(df_clean[col_for_box].dropna()) plt.ylabel(col_for_box) plt.title(f'Boxplot of {col_for_box} (After Cleaning)') plt.grid(True, alpha=0.3) # Plot 3: Correlation Heatmap if len(numeric_cols_clean) >= 2: plt.subplot(2, 2, 3) corr = df_clean[numeric_cols_clean].corr() sns.heatmap(corr, annot=True, cmap='coolwarm', center=0) plt.title('Correlation Matrix (After Cleaning)') # Plot 4: Bar chart or trend plt.subplot(2, 2, 4) cat_cols_clean = df_clean.select_dtypes(include=['object']).columns if len(cat_cols_clean) > 0: counts = df_clean[cat_cols_clean[0]].value_counts().head(10) plt.bar(range(len(counts)), counts.values) plt.xticks(range(len(counts)), counts.index, rotation=45, ha='right') plt.ylabel('Count') plt.title(f'Top values in {cat_cols_clean[0]}') elif len(numeric_cols_clean) >= 2: plt.plot(df_clean[numeric_cols_clean[0]].head(50).values, marker='o') plt.xlabel('Row Index') plt.ylabel(numeric_cols_clean[0]) plt.title(f'Trend of {numeric_cols_clean[0]} (First 50 rows)') plt.tight_layout() return report, fig # Create the interface demo = gr.Interface( fn=clean_and_analyze, inputs=gr.File(label="Upload CSV File", file_types=[".csv"]), outputs=[ gr.Textbox(label="Complete Analysis Report", lines=35), gr.Plot(label="Visualizations") ], title="Data Analysis Agent - With Automatic Data Cleaning", description="Upload any CSV file. The agent will: 1) Show missing values, 2) Clean/fill missing data, 3) Show analysis on cleaned data, 4) Generate visualizations." ) demo.launch()