Update app.py

app.py

@@ -2,7 +2,7 @@ import gradio as gr
 import pandas as pd
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.cluster import KMeans
-from sklearn.metrics import silhouette_score, silhouette_samples
+from sklearn.metrics import silhouette_score, silhouette_samples, davies_bouldin_score
 import matplotlib.pyplot as plt
 from sklearn.decomposition import PCA
 import re
@@ -10,12 +10,17 @@ from io import BytesIO
 import tempfile
 import numpy as np
 from PIL import Image
+from nltk.stem import WordNetLemmatizer
+from sklearn.preprocessing import normalize
 
 def preprocess_data(df):
     df.rename(columns={'Question Asked': 'texts'}, inplace=True)
     df['texts'] = df['texts'].astype(str)
     df['texts'] = df['texts'].str.lower()
     df['texts'] = df['texts'].apply(lambda text: re.sub(r'https?://\S+|www\.\S+', '', text))
+    
+    lemmatizer = WordNetLemmatizer()
+    df['texts'] = df['texts'].apply(lambda text: ' '.join([lemmatizer.lemmatize(word) for word in text.split()]))
 
     def remove_emoji(string):
         emoji_pattern = re.compile("["
@@ -111,8 +116,9 @@ def preprocess_data(df):
     return df
 
 def cluster_data(df, num_clusters):
-    vectorizer = TfidfVectorizer(stop_words='english')
+    vectorizer = TfidfVectorizer(stop_words='english', ngram_range=(1, 2), max_df=0.85, min_df=2)
     X = vectorizer.fit_transform(df['texts'])
+    X = normalize(X)
 
     kmeans = KMeans(n_clusters=num_clusters, random_state=0)
     kmeans.fit(X)
@@ -171,7 +177,25 @@ def silhouette_analysis(X, labels, num_clusters):
     img = Image.open(buf)
     return img
 
-def main(file, num_clusters_to_display):
+def find_optimal_clusters(X, max_clusters):
+    silhouette_scores = []
+    davies_bouldin_scores = []
+
+    for n_clusters in range(2, max_clusters + 1):
+        kmeans = KMeans(n_clusters=n_clusters, random_state=0)
+        labels = kmeans.fit_predict(X)
+        silhouette_avg = silhouette_score(X, labels)
+        davies_bouldin = davies_bouldin_score(X, labels)
+        
+        silhouette_scores.append(silhouette_avg)
+        davies_bouldin_scores.append(davies_bouldin)
+        
+        print(f"Clusters: {n_clusters}, Silhouette Score: {silhouette_avg}, Davies-Bouldin Index: {davies_bouldin}")
+
+    optimal_clusters = np.argmax(silhouette_scores) + 2
+    return optimal_clusters, silhouette_scores, davies_bouldin_scores
+
+def main(file, max_clusters_to_display):
     try:
         df = pd.read_csv(file)
         
@@ -179,7 +203,13 @@ def main(file, num_clusters_to_display):
         df = df[(df['Answer'] == 'Fallback Message shown')]
         
         df = preprocess_data(df)
-        df, X, kmeans = cluster_data(df, num_clusters=20)
+        vectorizer = TfidfVectorizer(stop_words='english', ngram_range=(1, 2), max_df=0.85, min_df=2)
+        X = vectorizer.fit_transform(df['texts'])
+        X = normalize(X)
+        
+        optimal_clusters, silhouette_scores, davies_bouldin_scores = find_optimal_clusters(X, max_clusters_to_display)
+        
+        df, X, kmeans = cluster_data(df, num_clusters=optimal_clusters)
         
         cluster_plot = visualize_clusters(df)
 
@@ -190,34 +220,37 @@ def main(file, num_clusters_to_display):
 
         # Filter out the largest cluster and get the next largest clusters
         largest_cluster = sorted_clusters[0]
-        filtered_clusters = sorted_clusters[1:num_clusters_to_display+1]
+        filtered_clusters = sorted_clusters[1:max_clusters_to_display+1]
 
         df = df[df['Cluster'].isin(filtered_clusters)]
         df['Cluster'] = pd.Categorical(df['Cluster'], categories=filtered_clusters, ordered=True)
         df = df.sort_values('Cluster')
 
         silhouette_avg = silhouette_score(X, kmeans.labels_)
-        silhouette_plot = silhouette_analysis(X, kmeans.labels_, num_clusters=15)
+        silhouette_plot = silhouette_analysis(X, kmeans.labels_, num_clusters=optimal_clusters)
+
+        davies_bouldin = davies_bouldin_score(X, kmeans.labels_)
 
         # Convert silhouette score to percentage
         silhouette_percentage = (silhouette_avg + 1) * 50
 
         with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as tmpfile:
             df.to_csv(tmpfile.name, index=False)
-            return tmpfile.name, silhouette_percentage, cluster_plot, silhouette_plot
+            return tmpfile.name, silhouette_percentage, davies_bouldin, cluster_plot, silhouette_plot
     except Exception as e:
         print(f"Error: {e}")
-        return str(e), None, None, None
+        return str(e), None, None, None, None
 
 interface = gr.Interface(
     fn=main,
     inputs=[
         gr.File(label="Upload CSV File (.csv)"),
-        gr.Slider(label="Number of Categories to Display", minimum=1, maximum=10, step=1, value=5)
+        gr.Slider(label="Max Clusters to Display", minimum=2, maximum 20, step=1, value=10)
     ],
     outputs=[
         gr.File(label="Clustered Data CSV"),
         gr.Number(label="Clustering Quality (%)"),
+        gr.Number(label="Davies-Bouldin Index"),
         gr.Image(label="Cluster Plot"),
         gr.Image(label="Silhouette Plot")
     ],