Fix

app.py

@@ -160,7 +160,7 @@ def display_results(orig_sentences,results,response_info,app_mode,model_name):
     main_sent = f"<div style=\"font-size:14px; color: #2f2f2f; text-align: left\">{response_info}<br/><br/></div>"
     main_sent += f"<div style=\"font-size:14px; color: #2f2f2f; text-align: left\">Showing results for model:&nbsp;<b>{model_name}</b></div>"
     score_text = "cosine distance"
-    main_sent += f"<div style=\"font-size:14px; color: #6f6f6f; text-align: left\">Clustering by {score_text}.&nbsp;<b>{len(results['clusters'])} clusters</b>.&nbsp;&nbsp;mean:{results['info']['mean']:.2f}&nbsp;std:{results['info']['std']:.2f}&nbsp;threshold hints:{str(results['info']['zscores'])}</div>"
+    main_sent += f"<div style=\"font-size:14px; color: #6f6f6f; text-align: left\">Clustering by {score_text}.&nbsp;<b>{len(results['clusters'])} clusters</b>.&nbsp;&nbsp;mean:{results['info']['mean']:.2f};&nbsp;std:{results['info']['std']:.2f};&nbsp;current threshold:{results['info']['current_threshold']}<br/>Threshold hints:{str(results['info']['zscores'])}<br/>Overlap stats(overlap,freq):{str(results['info']['overlap'])}</div>"
     body_sent = []
     download_data = {}
     for i in range(len(results["clusters"])):

twc_clustering.py

@@ -14,7 +14,7 @@ class TWCClustering:
         print("In Zscore  Clustering")
 
     def compute_matrix(self,embeddings):
-        print("Computing similarity matrix ...)")
+        #print("Computing similarity matrix ...)")
         embeddings= np.array(embeddings)
         start = time.time()
         vec_a = embeddings.T #vec_a shape (1024,)
@@ -23,7 +23,7 @@ class TWCClustering:
         similarity_matrix = np.inner(vec_a,vec_a)
         end = time.time()
         time_val = (end-start)*1000
-        print(f"Similarity matrix computation complete. Time taken:{(time_val/(1000*60)):.2f}  minutes")
+        #print(f"Similarity matrix computation complete. Time taken:{(time_val/(1000*60)):.2f}  minutes")
         return similarity_matrix
         
     def get_terms_above_threshold(self,matrix,embeddings,pivot_index,threshold):
@@ -63,6 +63,24 @@ class TWCClustering:
         return  {"pivot_index":center_index,"orig_index":pivot_index,"neighs":sorted_d}
          
 
+    def update_overlap_stats(self,overlap_dict,cluster_info):
+        arr = list(cluster_info["neighs"].keys())
+        for val in arr:
+            if (val not in overlap_dict):
+                overlap_dict[val] = 1
+            else:
+                overlap_dict[val] += 1
+
+    def bucket_overlap(self,overlap_dict):
+        bucket_dict = {}
+        for key in overlap_dict:
+            if (overlap_dict[key] not in bucket_dict):
+                bucket_dict[overlap_dict[key]] = 1
+            else:
+                bucket_dict[overlap_dict[key]] += 1
+        sorted_d = OrderedDict(sorted(bucket_dict.items(), key=lambda kv: kv[1], reverse=False))
+        return sorted_d
+        
     def cluster(self,output_file,texts,embeddings,threshold = 1.5):
         matrix = self.compute_matrix(embeddings)
         mean = np.mean(matrix)
@@ -71,13 +89,14 @@ class TWCClustering:
         inc = 0
         value = mean
         while (value < 1):
-            zscores.append(round(value,2))
+            zscores.append({"threshold":inc,"cosine":round(value,2)})
             inc += 1
             value = mean + inc*std
-        print("In clustering:",round(std,2),zscores)
+        #print("In clustering:",round(std,2),zscores)
         cluster_dict = {}
         cluster_dict["clusters"] = []
         picked_dict = {}
+        overlap_dict = {}
     
         for i in range(len(embeddings)):
             if (i in picked_dict):
@@ -86,8 +105,13 @@ class TWCClustering:
             arr = self.get_terms_above_threshold(matrix,embeddings,i,zscore)
             cluster_info = self.find_pivot_subgraph(i,arr,matrix,zscore)
             self.update_picked_dict(picked_dict,cluster_info["neighs"])
+            self.update_overlap_stats(overlap_dict,cluster_info)
             cluster_dict["clusters"].append(cluster_info)
-        cluster_dict["info"] ={"mean":mean,"std":std,"zscores":zscores}
+        curr_threshold = f"{threshold} (cosine:{mean+threshold*std:.2f})"
+        sorted_d = OrderedDict(sorted(overlap_dict.items(), key=lambda kv: kv[1], reverse=True))
+        #print(sorted_d)
+        sorted_d = self.bucket_overlap(overlap_dict)
+        cluster_dict["info"] ={"mean":mean,"std":std,"current_threshold":curr_threshold,"zscores":zscores,"overlap":list(sorted_d.items())}
         return cluster_dict