Update cross_encoder_reranking_train.py

cross_encoder_reranking_train.py

@@ -37,17 +37,8 @@ def cluster_and_rank(texts, threshold=0.8):
         return texts 
 
     embeddings = embed_text_list(texts)
-    similarity_matrix = np.dot(embeddings, np.transpose(embeddings))
-    max_sim = np.max(similarity_matrix)
-    distance_matrix = max_sim - similarity_matrix
-
-    clustering = AgglomerativeClustering(
-        n_clusters=None,
-        distance_threshold=max_sim-threshold,  # lower threshold = tighter clusters
-        metric='precomputed',
-        linkage='average'
-    )
-    labels = clustering.fit_predict(distance_matrix)
+    clustering = AgglomerativeClustering(n_clusters=None, distance_threshold=1-threshold, metric = "cosine", linkage='average')
+    labels = clustering.fit_predict(embeddings)
     
     clustered_texts = {}
     for label, text in zip(labels, texts):