Update app.py

app.py

@@ -107,8 +107,22 @@ def preprocess_data(df):
 
     return df
 
-def cluster_data(df):
-    num_clusters = 15  # Set the number of clusters to 15
+def elbow_method(X):
+    distortions = []
+    K = range(1, 20)
+    for k in K:
+        kmeanModel = KMeans(n_clusters=k)
+        kmeanModel.fit(X)
+        distortions.append(kmeanModel.inertia_)
+
+    plt.figure(figsize=(10, 6))
+    plt.plot(K, distortions, 'bx-')
+    plt.xlabel('k')
+    plt.ylabel('Distortion')
+    plt.title('The Elbow Method showing the optimal k')
+    plt.show()
+
+def cluster_data(df, num_clusters=15):
     vectorizer = TfidfVectorizer(stop_words='english')
     X = vectorizer.fit_transform(df['texts'])
 
@@ -117,54 +131,45 @@ def cluster_data(df):
     df['Cluster'] = kmeans.labels_
 
     pca = PCA(n_components=2)
-    principal_components = pca.fit_transform(X.toarray())
-    df['PCA1'] = principal_components[:, 0]
-    df['PCA2'] = principal_components[:, 1]
+    scatter_plot_points = pca.fit_transform(X.toarray())
+    colors = ['r', 'b', 'c', 'y', 'm', 'g']
 
-    return df
+    x_axis = [o[0] for o in scatter_plot_points]
+    y_axis = [o[1] for o in scatter_plot_points]
 
-def visualize_clusters(df):
     plt.figure(figsize=(10, 6))
-    scatter = plt.scatter(df['PCA1'], df['PCA2'], c=df['Cluster'], cmap='viridis')
-    plt.legend(*scatter.legend_elements(), title="Clusters")
-    plt.title('Clusters of User Queries')
-    plt.xlabel('PCA Component 1')
-    plt.ylabel('PCA Component 2')
+    plt.scatter(x_axis, y_axis, c=[colors[d] for d in kmeans.labels_])
     plt.show()
 
-def main(file, num_clusters_to_display):
-    try:
-        df = pd.read_excel(file)
-        df = preprocess_data(df)
-        df = cluster_data(df)
-        visualize_clusters(df)
+    return df
+
+def sort_and_filter_clusters(df, num_display_clusters):
+    cluster_sizes = df['Cluster'].value_counts().sort_values(ascending=False)
+    largest_clusters = cluster_sizes[cluster_sizes.index != 0].head(num_display_clusters).index
+    filtered_df = df[df['Cluster'].isin(largest_clusters)]
 
-        cluster_sizes = df['Cluster'].value_counts()
-        sorted_clusters = cluster_sizes.index.tolist()
-        df['Cluster'] = pd.Categorical(df['Cluster'], categories=sorted_clusters, ordered=True)
-        df = df.sort_values('Cluster')
+    return filtered_df
 
-        # Filter out cluster 0 and get the largest clusters
-        filtered_clusters = [cluster for cluster in sorted_clusters if cluster != 0]
-        top_clusters = filtered_clusters[:num_clusters_to_display]
+def main(file, num_display_clusters):
+    df = pd.read_excel(file)
+    df = preprocess_data(df)
+    df = cluster_data(df, num_clusters=15)
+    df = sort_and_filter_clusters(df, num_display_clusters)
+    
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as tmp:
+        df.to_csv(tmp.name, index=False)
+        tmp.seek(0)
+        return tmp.name
 
-        df = df[df['Cluster'].isin(top_clusters)]
-        df['Cluster'] = pd.Categorical(df['Cluster'], categories=top_clusters, ordered=True)
-        df = df.sort_values('Cluster')
+input_file = gr.inputs.File(label="Upload an Excel file (.xlsx)")
+num_display_clusters = gr.inputs.Slider(minimum=1, maximum=10, step=1, default=5, label="Number of Largest Clusters to Display")
 
-        with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as tmpfile:
-            df.to_csv(tmpfile.name, index=False)
-            return tmpfile.name
-    except Exception as e:
-        return str(e)
+output_file = gr.outputs.File(label="Clustered Data CSV")
 
 interface = gr.Interface(
     fn=main,
-    inputs=[
-        gr.File(label="Upload Excel File (.xlsx)"),
-        gr.Slider(1, 10, step=1, label="Number of Largest Clusters to Display")
-    ],
-    outputs=gr.File(label="Clustered Data CSV"),
+    inputs=[input_file, num_display_clusters],
+    outputs=output_file,
     title="Unanswered User Queries Clustering",
     description="Upload an Excel file (.xlsx) and select the number of largest clusters to display (excluding cluster 0)"
 )