updates

utils/clustering_utils.py

@@ -139,6 +139,15 @@ def clustering_author(background_corpus_df: pd.DataFrame,
         background_corpus_df['cluster_label'] = final_labels_for_df
         return background_corpus_df
 
+    # For cosine metric, normalize embeddings to unit length.
+    # This is standard practice as cosine similarity is equivalent to Euclidean
+    # distance on L2-normalized vectors. DBSCAN's 'cosine' metric internally
+    # works with these normalized distances.
+    if metric == 'cosine':
+        from sklearn.preprocessing import normalize
+        print("Normalizing embeddings for cosine distance...")
+        X = normalize(X, norm='l2', axis=1)
+
     if eps_values is None:
         if metric == 'cosine':
             eps_values = [0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8]