Update app.py

app.py

@@ -1,7 +1,8 @@
 import pandas as pd
 import streamlit as st
 from difflib import SequenceMatcher
-
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
 
 ms = st.session_state
 if "themes" not in ms: 
@@ -62,16 +63,29 @@ def find_exact_matches(df1, df2, column_name):
 def find_similar_texts(df1, df2, column_name, exact_matches, threshold=0.8):
     # Find rows with similar texts in the specified column, excluding exact matches
     similar_texts = []
-    for index1, row1 in df1.iterrows():
-        for index2, row2 in df2.iterrows():
-            if (index1, index2) not in exact_matches:
-                similarity = SequenceMatcher(None, str(row1[column_name]), str(row2[column_name])).ratio()
-                if similarity >= threshold:
-                    similar_texts.append((index1, index2, row1[column_name], row2[column_name]))
+    exact_match_indices = set(exact_matches.index.tolist())
+    
+    # Concatenate texts from both dataframes
+    all_texts = df1[column_name].tolist() + df2[column_name].tolist()
+    
+    # Compute TF-IDF vectors
+    vectorizer = TfidfVectorizer()
+    tfidf_matrix = vectorizer.fit_transform(all_texts)
+    
+    # Compute cosine similarity matrix
+    similarity_matrix = cosine_similarity(tfidf_matrix, tfidf_matrix)
+    
+    # Iterate over pairs of rows to find similar texts
+    for i, row1 in df1.iterrows():
+        for j, row2 in df2.iterrows():
+            if i not in exact_match_indices and j not in exact_match_indices:
+                similarity = similarity_matrix[i, len(df1) + j]
+                if similarity >= threshold and similarity < 1:  # Exclude exact matches
+                    similar_texts.append((i, j, row1[column_name], row2[column_name]))
+    
     return similar_texts
 
 
-
 def main():
     st.title("Item Comparison App")