Update app.py

app.py

@@ -4,114 +4,141 @@ import gradio as gr
 from sklearn.model_selection import train_test_split
 from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
 from sklearn.metrics import confusion_matrix, classification_report, mean_squared_error, r2_score
-from sklearn.cluster import KMeans
+from sklearn.cluster import KMeans, AgglomerativeClustering
 from sklearn.decomposition import PCA
+from sklearn.preprocessing import StandardScaler
+from sklearn.feature_selection import f_classif
 import matplotlib.pyplot as plt
 import seaborn as sns
 import io
 
-def analyze_csv(file, label_col):
+def analyze_csv(file, label_col, n_clusters):
     try:
         df = pd.read_csv(file.name if hasattr(file, "name") else file)
     except Exception as e:
-        return None, None, None, None, f"error reading csv: {e}"
+        return (None,)*6 + (f"error reading csv: {e}",)
     
     if label_col not in df.columns:
-        return None, None, None, None, f"label column '{label_col}' not in data"
+        return (None,)*6 + (f"label column '{label_col}' not in data",)
     
     df = df.dropna()
-    X = df.drop(columns=[label_col])
+    # separate target and features
     y = df[label_col]
+    X = df.drop(columns=[label_col])
+    # create one-hot encodings for non-numeric columns
+    X_processed = pd.get_dummies(X)
     
-    # use only numeric features for modeling; drop non-numeric columns
-    X_numeric = X.select_dtypes(include=[np.number])
-    if X_numeric.shape[1] == 0:
-        return None, None, None, None, "no numeric features available for modeling"
+    # scale features for clustering methods
+    scaler = StandardScaler()
+    X_scaled = scaler.fit_transform(X_processed)
     
     results_text = ""
-    cm_img = None
-    reg_img = None
-    fi_img = None
-    
-    # placeholder for feature importances
-    feature_importances = None
+    model_img = None
 
-    # if label is numeric, treat as regression; otherwise, classification
+    # model training & evaluation: regression if y numeric, classification otherwise
     if pd.api.types.is_numeric_dtype(y):
-        task = "regression"
-        X_train, X_test, y_train, y_test = train_test_split(X_numeric, y, test_size=0.3, random_state=42)
+        # regression
+        X_train, X_test, y_train, y_test = train_test_split(X_processed, y, test_size=0.3, random_state=42)
         model = RandomForestRegressor(random_state=42)
         model.fit(X_train, y_train)
         y_pred = model.predict(X_test)
         mse = mean_squared_error(y_test, y_pred)
         r2 = r2_score(y_test, y_pred)
         results_text += f"regression results:\nmse: {mse:.3f}\nr2: {r2:.3f}\n"
-        # regression scatter plot: true vs predicted
-        plt.figure()
+        # scatter plot: true vs predicted with y=x line
+        plt.figure(figsize=(6,4))
         plt.scatter(y_test, y_pred, alpha=0.7)
+        plt.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'r--')
         plt.xlabel("true values")
         plt.ylabel("predicted values")
         plt.title("regression: true vs predicted")
         buf = io.BytesIO()
-        plt.savefig(buf, format="png")
+        plt.savefig(buf, format="png", bbox_inches="tight")
         plt.close()
         buf.seek(0)
-        reg_img = buf
-        # note: confusion matrix not applicable here
-        feature_importances = model.feature_importances_
+        model_img = buf
     else:
-        task = "classification"
-        # encode labels as integers
-        y_encoded = pd.factorize(y)[0]
-        X_train, X_test, y_train, y_test = train_test_split(X_numeric, y_encoded, test_size=0.3, random_state=42)
+        # classification
+        y_encoded, uniques = pd.factorize(y)
+        X_train, X_test, y_train, y_test = train_test_split(X_processed, y_encoded, test_size=0.3, random_state=42)
         model = RandomForestClassifier(random_state=42)
         model.fit(X_train, y_train)
         y_pred = model.predict(X_test)
         cm = confusion_matrix(y_test, y_pred)
         cr = classification_report(y_test, y_pred)
         results_text += f"classification results:\n{cr}\n"
-        plt.figure()
+        plt.figure(figsize=(6,4))
         sns.heatmap(cm, annot=True, fmt="d", cmap="Blues")
         plt.xlabel("predicted")
         plt.ylabel("true")
         plt.title("confusion matrix")
         buf = io.BytesIO()
-        plt.savefig(buf, format="png")
+        plt.savefig(buf, format="png", bbox_inches="tight")
         plt.close()
         buf.seek(0)
-        cm_img = buf
-        feature_importances = model.feature_importances_
-    
-    # feature importance plot
-    fi = pd.Series(feature_importances, index=X_numeric.columns).sort_values(ascending=False)
+        model_img = buf
+
+    # feature importance plot (from the model)
+    fi = pd.Series(model.feature_importances_, index=X_processed.columns).sort_values(ascending=False)
     plt.figure(figsize=(8,4))
     sns.barplot(x=fi.values, y=fi.index)
     plt.title("feature importances")
+    plt.xlabel("importance")
+    plt.ylabel("feature")
     buf = io.BytesIO()
     plt.savefig(buf, format="png", bbox_inches="tight")
     plt.close()
     buf.seek(0)
     fi_img = buf
 
-    # clustering: kmeans on numeric features; use pca for 2d visualization
-    k = 3
-    kmeans = KMeans(n_clusters=k, random_state=42)
-    clusters = kmeans.fit_predict(X_numeric)
+    # clustering with kmeans
+    kmeans = KMeans(n_clusters=n_clusters, random_state=42)
+    clusters_kmeans = kmeans.fit_predict(X_scaled)
     pca = PCA(n_components=2, random_state=42)
-    X_pca = pca.fit_transform(X_numeric)
-    plt.figure()
-    scatter = plt.scatter(X_pca[:,0], X_pca[:,1], c=clusters, cmap="viridis", alpha=0.7)
+    X_pca = pca.fit_transform(X_scaled)
+    plt.figure(figsize=(6,4))
+    scatter = plt.scatter(X_pca[:,0], X_pca[:,1], c=clusters_kmeans, cmap="viridis", alpha=0.7)
+    plt.xlabel("pca 1")
+    plt.ylabel("pca 2")
+    plt.title(f"kmeans clustering (k={n_clusters})")
+    plt.colorbar(scatter, ticks=range(n_clusters))
+    buf = io.BytesIO()
+    plt.savefig(buf, format="png", bbox_inches="tight")
+    plt.close()
+    buf.seek(0)
+    kmeans_img = buf
+
+    # clustering with agglomerative clustering
+    agg = AgglomerativeClustering(n_clusters=n_clusters)
+    clusters_agg = agg.fit_predict(X_scaled)
+    plt.figure(figsize=(6,4))
+    scatter = plt.scatter(X_pca[:,0], X_pca[:,1], c=clusters_agg, cmap="plasma", alpha=0.7)
     plt.xlabel("pca 1")
     plt.ylabel("pca 2")
-    plt.title("kmeans clustering (k=3)")
-    plt.colorbar(scatter, ticks=range(k))
+    plt.title(f"agglomerative clustering (k={n_clusters})")
+    plt.colorbar(scatter, ticks=range(n_clusters))
+    buf = io.BytesIO()
+    plt.savefig(buf, format="png", bbox_inches="tight")
+    plt.close()
+    buf.seek(0)
+    agg_img = buf
+
+    # differentiating features among clusters (using kmeans clusters)
+    f_scores, p_vals = f_classif(X_processed, clusters_kmeans)
+    f_series = pd.Series(f_scores, index=X_processed.columns).sort_values(ascending=False)
+    top_features = f_series.head(10)
+    plt.figure(figsize=(8,4))
+    sns.barplot(x=top_features.values, y=top_features.index, palette="mako")
+    plt.title("top differentiating features (anova f-scores)")
+    plt.xlabel("f-score")
+    plt.ylabel("feature")
     buf = io.BytesIO()
     plt.savefig(buf, format="png", bbox_inches="tight")
     plt.close()
     buf.seek(0)
-    cluster_img = buf
+    diff_img = buf
 
-    return cm_img, reg_img, fi_img, cluster_img, results_text
+    return results_text, model_img, fi_img, kmeans_img, agg_img, diff_img
 
 def update_dropdown(file):
     try:
@@ -125,23 +152,26 @@ with gr.Blocks() as demo:
     with gr.Row():
         file_input = gr.File(label="upload csv", file_types=[".csv"])
         label_dropdown = gr.Dropdown(label="select label column", choices=[])
+        clusters_slider = gr.Slider(minimum=2, maximum=10, step=1, value=3, label="number of clusters")
     
     file_input.change(fn=update_dropdown, inputs=file_input, outputs=label_dropdown)
     
     analyze_btn = gr.Button("analyze")
     with gr.Tabs():
         with gr.TabItem("results"):
-            results_textbox = gr.Textbox(label="metrics & results", lines=10)
-        with gr.TabItem("confusion matrix"):
-            cm_output = gr.Image(label="confusion matrix")
-        with gr.TabItem("regression plot"):
-            reg_output = gr.Image(label="regression plot")
+            results_textbox = gr.Textbox(label="metrics & descriptions", lines=10)
+        with gr.TabItem("model visualization"):
+            model_img_output = gr.Image(label="model output (confusion matrix or regression scatter)")
         with gr.TabItem("feature importances"):
             fi_output = gr.Image(label="feature importances")
-        with gr.TabItem("clustering"):
-            cluster_output = gr.Image(label="cluster plot")
+        with gr.TabItem("kmeans clustering"):
+            kmeans_output = gr.Image(label="kmeans clustering (pca projection)")
+        with gr.TabItem("agglomerative clustering"):
+            agg_output = gr.Image(label="agglomerative clustering (pca projection)")
+        with gr.TabItem("cluster differentiation"):
+            diff_output = gr.Image(label="differentiating features among clusters")
     
-    analyze_btn.click(fn=analyze_csv, inputs=[file_input, label_dropdown],
-                      outputs=[cm_output, reg_output, fi_output, cluster_output, results_textbox])
+    analyze_btn.click(fn=analyze_csv, inputs=[file_input, label_dropdown, clusters_slider],
+                      outputs=[results_textbox, model_img_output, fi_output, kmeans_output, agg_output, diff_output])
 
 demo.launch()