Updated app with code for deduplication

app.py

@@ -10,12 +10,15 @@ import tqdm
 # Load the model at startup
 model = StaticModel.from_pretrained("minishlab/M2V_base_output")
 
-# Load the default datasets at startup
-default_dataset1_name = "ag_news"
+# Update default dataset to 'sst2' and set default threshold to 0.9
+default_dataset1_name = "sst2"
 default_dataset1_split = "train"
-default_dataset2_name = "ag_news"
-default_dataset2_split = "test"
+default_dataset2_name = "sst2"
+default_dataset2_split = "validation"
+default_text_column = "sentence"
+default_threshold = 0.9
 
+# Load the default datasets at startup
 ds_default1 = load_dataset(default_dataset1_name, split=default_dataset1_split)
 ds_default2 = load_dataset(default_dataset2_name, split=default_dataset2_split)
 
@@ -23,20 +26,28 @@ def deduplicate(embedding_matrix: np.ndarray, threshold: float, batch_size: int
     """
     Deduplicate embeddings and return the deduplicated indices and a mapping of removed indices to their corresponding original indices.
     """
-    reach = Reach(vectors=embedding_matrix, items=[str(i) for i in range(len(embedding_matrix))])
+    # Informative progress bar for building the index
+    progress.tqdm.write("Building search index...")
+    with progress.tqdm(total=1, desc="Building index") as p:
+        reach = Reach(vectors=embedding_matrix, items=[str(i) for i in range(len(embedding_matrix))])
+        p.update(1)
 
     deduplicated_indices = set(range(len(embedding_matrix)))
     duplicate_to_original_mapping = {}
 
+    # Informative progress bar for nearest neighbor search
+    progress.tqdm.write("Finding nearest neighbors...")
     results = reach.nearest_neighbor_threshold(
         embedding_matrix,
         threshold=threshold,
         batch_size=batch_size,
-        show_progressbar=True  # Allow internal progress bar
+        show_progressbar=False  # Disable internal progress bar
     )
 
-    # Process duplicates
-    for i, similar_items in enumerate(progress.tqdm(results, desc="Processing duplicates", total=len(embedding_matrix))):
+    total_items = len(embedding_matrix)
+    # Processing duplicates with a progress bar
+    progress.tqdm.write("Processing duplicates...")
+    for i, similar_items in enumerate(progress.tqdm(results, desc="Processing duplicates", total=total_items)):
         if i not in deduplicated_indices:
             continue
 
@@ -53,19 +64,28 @@ def deduplicate_across_datasets(embedding_matrix_1: np.ndarray, embedding_matrix
     """
     Deduplicate embeddings across two datasets and return the indices of duplicates between them.
     """
-    reach = Reach(vectors=embedding_matrix_1, items=[str(i) for i in range(len(embedding_matrix_1))])
+    # Informative progress bar for building the index
+    progress.tqdm.write("Building search index from Dataset 1...")
+    with progress.tqdm(total=1, desc="Building index for Dataset 1") as p:
+        reach = Reach(vectors=embedding_matrix_1, items=[str(i) for i in range(len(embedding_matrix_1))])
+        p.update(1)
 
     duplicate_indices_in_test = []
     duplicate_to_original_mapping = {}
 
+    # Informative progress bar for nearest neighbor search
+    progress.tqdm.write("Finding nearest neighbors between datasets...")
     results = reach.nearest_neighbor_threshold(
         embedding_matrix_2,
         threshold=threshold,
         batch_size=batch_size,
-        show_progressbar=True  # Allow internal progress bar
+        show_progressbar=False  # Disable internal progress bar
     )
 
-    for i, similar_items in enumerate(progress.tqdm(results, desc="Processing duplicates across datasets", total=len(embedding_matrix_2))):
+    total_items = len(embedding_matrix_2)
+    # Processing duplicates with a progress bar
+    progress.tqdm.write("Processing duplicates across datasets...")
+    for i, similar_items in enumerate(progress.tqdm(results, desc="Processing duplicates across datasets", total=total_items)):
         similar_indices = [int(item[0]) for item in similar_items if item[1] >= threshold]
 
         if similar_indices:
@@ -86,7 +106,7 @@ def perform_deduplication(
     dataset2_name="",
     dataset2_split="",
     dataset2_text_column="",
-    threshold=0.8,
+    threshold=default_threshold,
     progress=gr.Progress(track_tqdm=True)
 ):
     # Monkey-patch tqdm
@@ -102,89 +122,63 @@ def perform_deduplication(
         threshold = float(threshold)
         
         if deduplication_type == "Single dataset":
-            # Check if the dataset is the default one
+            # Load Dataset 1
             if dataset1_name == default_dataset1_name and dataset1_split == default_dataset1_split:
                 ds = ds_default1
             else:
                 ds = load_dataset(dataset1_name, split=dataset1_split)
             
-            # Extract texts
-            texts = [example[dataset1_text_column] for example in ds]
+            # Extract texts with progress bar
+            progress.tqdm.write("Extracting texts from Dataset 1...")
+            texts = [example[dataset1_text_column] for example in progress.tqdm(ds, desc="Extracting texts", total=len(ds))]
             
-            # Compute embeddings
-            embedding_matrix = model.encode(texts, show_progressbar=True)  # Enable internal progress bar
+            # Compute embeddings with progress bar
+            progress.tqdm.write("Computing embeddings for Dataset 1...")
+            embedding_matrix = model.encode(texts, show_progressbar=False)  # Disable internal progress bar
+            embedding_matrix = progress.tqdm(embedding_matrix, desc="Computing embeddings", total=len(texts))
             
             # Deduplicate
-            deduplicated_indices, duplicate_to_original_mapping = deduplicate(embedding_matrix, threshold, progress=progress)
-            
-            # Prepare the results
-            num_duplicates = len(duplicate_to_original_mapping)
-            num_total = len(texts)
-            num_deduplicated = len(deduplicated_indices)
-            
-            result_text = f"**Total documents:** {num_total}\n"
-            result_text += f"**Number of duplicates found:** {num_duplicates}\n"
-            result_text += f"**Number of unique documents after deduplication:** {num_deduplicated}\n\n"
-            
-            # Show deduplicated examples
-            result_text += "**Examples of duplicates found:**\n\n"
-            num_examples = min(5, num_duplicates)
-            for duplicate_idx, original_idx in list(duplicate_to_original_mapping.items())[:num_examples]:
-                original_text = texts[original_idx]
-                duplicate_text = texts[duplicate_idx]
-                differences = display_word_differences(original_text, duplicate_text)
-                result_text += f"**Original text:**\n{original_text}\n\n"
-                result_text += f"**Duplicate text:**\n{duplicate_text}\n\n"
-                result_text += f"**Differences:**\n{differences}\n"
-                result_text += "-" * 50 + "\n\n"
+            result_text = deduplicate_and_prepare_results_single(
+                embedding_matrix, texts, threshold, progress
+            )
             
             return result_text
                 
         elif deduplication_type == "Cross-dataset":
-            # Dataset 1
+            # Load Dataset 1
             if dataset1_name == default_dataset1_name and dataset1_split == default_dataset1_split:
                 ds1 = ds_default1
             else:
                 ds1 = load_dataset(dataset1_name, split=dataset1_split)
             
-            # Dataset 2
+            # Load Dataset 2
             if dataset2_name == default_dataset2_name and dataset2_split == default_dataset2_split:
                 ds2 = ds_default2
             else:
                 ds2 = load_dataset(dataset2_name, split=dataset2_split)
             
-            # Extract texts
-            texts1 = [example[dataset1_text_column] for example in ds1]
-            texts2 = [example[dataset2_text_column] for example in ds2]
+            # Extract texts from Dataset 1
+            progress.tqdm.write("Extracting texts from Dataset 1...")
+            texts1 = [example[dataset1_text_column] for example in progress.tqdm(ds1, desc="Extracting texts from Dataset 1", total=len(ds1))]
+            
+            # Extract texts from Dataset 2
+            progress.tqdm.write("Extracting texts from Dataset 2...")
+            texts2 = [example[dataset2_text_column] for example in progress.tqdm(ds2, desc="Extracting texts from Dataset 2", total=len(ds2))]
             
-            # Compute embeddings
-            embedding_matrix1 = model.encode(texts1, show_progressbar=True)  # Enable internal progress bar
-            embedding_matrix2 = model.encode(texts2, show_progressbar=True)  # Enable internal progress bar
+            # Compute embeddings for Dataset 1
+            progress.tqdm.write("Computing embeddings for Dataset 1...")
+            embedding_matrix1 = model.encode(texts1, show_progressbar=False)
+            embedding_matrix1 = progress.tqdm(embedding_matrix1, desc="Computing embeddings for Dataset 1", total=len(texts1))
+            
+            # Compute embeddings for Dataset 2
+            progress.tqdm.write("Computing embeddings for Dataset 2...")
+            embedding_matrix2 = model.encode(texts2, show_progressbar=False)
+            embedding_matrix2 = progress.tqdm(embedding_matrix2, desc="Computing embeddings for Dataset 2", total=len(texts2))
             
             # Deduplicate across datasets
-            duplicate_indices_in_ds2, duplicate_to_original_mapping = deduplicate_across_datasets(
-                embedding_matrix1, embedding_matrix2, threshold, progress=progress)
-            
-            num_duplicates = len(duplicate_indices_in_ds2)
-            num_total_ds2 = len(texts2)
-            num_unique_ds2 = num_total_ds2 - num_duplicates
-            
-            result_text = f"**Total documents in {dataset2_name}/{dataset2_split}:** {num_total_ds2}\n"
-            result_text += f"**Number of duplicates found in {dataset2_name}/{dataset2_split}:** {num_duplicates}\n"
-            result_text += f"**Number of unique documents in {dataset2_name}/{dataset2_split} after deduplication:** {num_unique_ds2}\n\n"
-            
-            # Show deduplicated examples
-            result_text += "**Examples of duplicates found in Dataset 2:**\n\n"
-            num_examples = min(5, num_duplicates)
-            for duplicate_idx in duplicate_indices_in_ds2[:num_examples]:
-                original_idx = duplicate_to_original_mapping[duplicate_idx]
-                original_text = texts1[original_idx]
-                duplicate_text = texts2[duplicate_idx]
-                differences = display_word_differences(original_text, duplicate_text)
-                result_text += f"**Original text (Dataset 1):**\n{original_text}\n\n"
-                result_text += f"**Duplicate text (Dataset 2):**\n{duplicate_text}\n\n"
-                result_text += f"**Differences:**\n{differences}\n"
-                result_text += "-" * 50 + "\n\n"
+            result_text = deduplicate_and_prepare_results_cross(
+                embedding_matrix1, embedding_matrix2, texts1, texts2, threshold, progress, dataset2_name, dataset2_split
+            )
             
             return result_text
 
@@ -200,52 +194,116 @@ def perform_deduplication(
         else:
             del Reach.tqdm  # If it wasn't originally in Reach's __dict__
 
+def deduplicate_and_prepare_results_single(embedding_matrix, texts, threshold, progress):
+    # Deduplicate
+    deduplicated_indices, duplicate_to_original_mapping = deduplicate(
+        embedding_matrix, threshold, progress=progress
+    )
+    
+    # Prepare the results
+    num_duplicates = len(duplicate_to_original_mapping)
+    num_total = len(texts)
+    num_deduplicated = len(deduplicated_indices)
+    
+    result_text = f"**Total documents:** {num_total}\n"
+    result_text += f"**Number of duplicates found:** {num_duplicates}\n"
+    result_text += f"**Number of unique documents after deduplication:** {num_deduplicated}\n\n"
+    
+    # Show deduplicated examples
+    if num_duplicates > 0:
+        result_text += "**Examples of duplicates found:**\n\n"
+        num_examples = min(5, num_duplicates)
+        for duplicate_idx, original_idx in list(duplicate_to_original_mapping.items())[:num_examples]:
+            original_text = texts[original_idx]
+            duplicate_text = texts[duplicate_idx]
+            differences = display_word_differences(original_text, duplicate_text)
+            result_text += f"**Original text:**\n{original_text}\n\n"
+            result_text += f"**Duplicate text:**\n{duplicate_text}\n\n"
+            result_text += f"**Differences:**\n{differences}\n"
+            result_text += "-" * 50 + "\n\n"
+    else:
+        result_text += "No duplicates found."
+    
+    return result_text
+
+def deduplicate_and_prepare_results_cross(embedding_matrix1, embedding_matrix2, texts1, texts2, threshold, progress, dataset2_name, dataset2_split):
+    # Deduplicate across datasets
+    duplicate_indices_in_ds2, duplicate_to_original_mapping = deduplicate_across_datasets(
+        embedding_matrix1, embedding_matrix2, threshold, progress=progress
+    )
+    
+    num_duplicates = len(duplicate_indices_in_ds2)
+    num_total_ds2 = len(texts2)
+    num_unique_ds2 = num_total_ds2 - num_duplicates
+    
+    result_text = f"**Total documents in {dataset2_name}/{dataset2_split}:** {num_total_ds2}\n"
+    result_text += f"**Number of duplicates found in {dataset2_name}/{dataset2_split}:** {num_duplicates}\n"
+    result_text += f"**Number of unique documents in {dataset2_name}/{dataset2_split} after deduplication:** {num_unique_ds2}\n\n"
+    
+    # Show deduplicated examples
+    if num_duplicates > 0:
+        result_text += "**Examples of duplicates found in Dataset 2:**\n\n"
+        num_examples = min(5, num_duplicates)
+        for duplicate_idx in duplicate_indices_in_ds2[:num_examples]:
+            original_idx = duplicate_to_original_mapping[duplicate_idx]
+            original_text = texts1[original_idx]
+            duplicate_text = texts2[duplicate_idx]
+            differences = display_word_differences(original_text, duplicate_text)
+            result_text += f"**Original text (Dataset 1):**\n{original_text}\n\n"
+            result_text += f"**Duplicate text (Dataset 2):**\n{duplicate_text}\n\n"
+            result_text += f"**Differences:**\n{differences}\n"
+            result_text += "-" * 50 + "\n\n"
+    else:
+        result_text += "No duplicates found."
+    
+    return result_text
+
 with gr.Blocks() as demo:
     gr.Markdown("# Semantic Deduplication")
-    
+
     deduplication_type = gr.Radio(
         choices=["Single dataset", "Cross-dataset"],
         label="Deduplication Type",
         value="Single dataset"
     )
-    
+
     with gr.Row():
-        dataset1_name = gr.Textbox(value="ag_news", label="Dataset 1 Name")
-        dataset1_split = gr.Textbox(value="train", label="Dataset 1 Split")
-        dataset1_text_column = gr.Textbox(value="text", label="Text Column Name")
-    
+        dataset1_name = gr.Textbox(value=default_dataset1_name, label="Dataset 1 Name")
+        dataset1_split = gr.Textbox(value=default_dataset1_split, label="Dataset 1 Split")
+        dataset1_text_column = gr.Textbox(value=default_text_column, label="Text Column Name")
+
     dataset2_inputs = gr.Column(visible=False)
     with dataset2_inputs:
         gr.Markdown("### Dataset 2")
         with gr.Row():
-            dataset2_name = gr.Textbox(value="ag_news", label="Dataset 2 Name")
-            dataset2_split = gr.Textbox(value="test", label="Dataset 2 Split")
-            dataset2_text_column = gr.Textbox(value="text", label="Text Column Name")
-    
+            dataset2_name = gr.Textbox(value=default_dataset2_name, label="Dataset 2 Name")
+            dataset2_split = gr.Textbox(value=default_dataset2_split, label="Dataset 2 Split")
+            dataset2_text_column = gr.Textbox(value=default_text_column, label="Text Column Name")
+
     threshold = gr.Slider(
         minimum=0.0,
         maximum=1.0,
-        value=0.8,
+        value=default_threshold,
         label="Similarity Threshold"
     )
-    
+
     compute_button = gr.Button("Compute")
-    
+
     output = gr.Markdown()
-    
+
     # Function to update the visibility of dataset2_inputs
     def update_visibility(deduplication_type_value):
         if deduplication_type_value == "Cross-dataset":
             return gr.update(visible=True)
         else:
             return gr.update(visible=False)
-    
+
     deduplication_type.change(
         update_visibility,
         inputs=deduplication_type,
         outputs=dataset2_inputs
     )
-    
+
     compute_button.click(
         fn=perform_deduplication,
         inputs=[
@@ -302,7 +360,7 @@ demo.launch()
 #     )
 
 #     # Process duplicates
-#     for i, similar_items in enumerate(progress.tqdm(results, desc="Processing duplicates")):
+#     for i, similar_items in enumerate(progress.tqdm(results, desc="Processing duplicates", total=len(embedding_matrix))):
 #         if i not in deduplicated_indices:
 #             continue
 
@@ -331,8 +389,7 @@ demo.launch()
 #         show_progressbar=True  # Allow internal progress bar
 #     )
 
-#     # Process duplicates
-#     for i, similar_items in enumerate(progress.tqdm(results, desc="Processing duplicates across datasets")):
+#     for i, similar_items in enumerate(progress.tqdm(results, desc="Processing duplicates across datasets", total=len(embedding_matrix_2))):
 #         similar_indices = [int(item[0]) for item in similar_items if item[1] >= threshold]
 
 #         if similar_indices:
@@ -358,9 +415,11 @@ demo.launch()
 # ):
 #     # Monkey-patch tqdm
 #     original_tqdm = tqdm.tqdm
+#     original_reach_tqdm = Reach.__dict__['tqdm'] if 'tqdm' in Reach.__dict__ else None
 #     tqdm.tqdm = progress.tqdm
 #     sys.modules['tqdm'].tqdm = progress.tqdm
 #     sys.modules['tqdm.auto'].tqdm = progress.tqdm
+#     Reach.tqdm = progress.tqdm  # Monkey-patch reach's tqdm
 
 #     try:
 #         # Convert threshold to float
@@ -427,7 +486,8 @@ demo.launch()
 #             embedding_matrix2 = model.encode(texts2, show_progressbar=True)  # Enable internal progress bar
             
 #             # Deduplicate across datasets
-#             duplicate_indices_in_ds2, duplicate_to_original_mapping = deduplicate_across_datasets(embedding_matrix1, embedding_matrix2, threshold, progress=progress)
+#             duplicate_indices_in_ds2, duplicate_to_original_mapping = deduplicate_across_datasets(
+#                 embedding_matrix1, embedding_matrix2, threshold, progress=progress)
             
 #             num_duplicates = len(duplicate_indices_in_ds2)
 #             num_total_ds2 = len(texts2)
@@ -458,6 +518,12 @@ demo.launch()
 #         sys.modules['tqdm'].tqdm = original_tqdm
 #         sys.modules['tqdm.auto'].tqdm = original_tqdm
 
+#         # Restore reach's original tqdm
+#         if original_reach_tqdm is not None:
+#             Reach.tqdm = original_reach_tqdm
+#         else:
+#             del Reach.tqdm  # If it wasn't originally in Reach's __dict__
+
 # with gr.Blocks() as demo:
 #     gr.Markdown("# Semantic Deduplication")
     
@@ -520,3 +586,261 @@ demo.launch()
 #     )
     
 # demo.launch()
+
+
+# # import gradio as gr
+# # from datasets import load_dataset
+# # import numpy as np
+# # from model2vec import StaticModel
+# # from reach import Reach
+# # from difflib import ndiff
+# # import sys
+# # import tqdm
+
+# # # Load the model at startup
+# # model = StaticModel.from_pretrained("minishlab/M2V_base_output")
+
+# # # Load the default datasets at startup
+# # default_dataset1_name = "ag_news"
+# # default_dataset1_split = "train"
+# # default_dataset2_name = "ag_news"
+# # default_dataset2_split = "test"
+
+# # ds_default1 = load_dataset(default_dataset1_name, split=default_dataset1_split)
+# # ds_default2 = load_dataset(default_dataset2_name, split=default_dataset2_split)
+
+# # def deduplicate(embedding_matrix: np.ndarray, threshold: float, batch_size: int = 1024, progress=None) -> tuple[np.ndarray, dict[int, int]]:
+# #     """
+# #     Deduplicate embeddings and return the deduplicated indices and a mapping of removed indices to their corresponding original indices.
+# #     """
+# #     reach = Reach(vectors=embedding_matrix, items=[str(i) for i in range(len(embedding_matrix))])
+
+# #     deduplicated_indices = set(range(len(embedding_matrix)))
+# #     duplicate_to_original_mapping = {}
+
+# #     results = reach.nearest_neighbor_threshold(
+# #         embedding_matrix,
+# #         threshold=threshold,
+# #         batch_size=batch_size,
+# #         show_progressbar=True  # Allow internal progress bar
+# #     )
+
+# #     # Process duplicates
+# #     for i, similar_items in enumerate(progress.tqdm(results, desc="Processing duplicates")):
+# #         if i not in deduplicated_indices:
+# #             continue
+
+# #         similar_indices = [int(item[0]) for item in similar_items if int(item[0]) != i]
+
+# #         for sim_idx in similar_indices:
+# #             if sim_idx in deduplicated_indices:
+# #                 deduplicated_indices.remove(sim_idx)
+# #                 duplicate_to_original_mapping[sim_idx] = i
+
+# #     return np.array(list(deduplicated_indices)), duplicate_to_original_mapping
+
+# # def deduplicate_across_datasets(embedding_matrix_1: np.ndarray, embedding_matrix_2: np.ndarray, threshold: float, batch_size: int = 1024, progress=None) -> tuple[list[int], dict[int, int]]:
+# #     """
+# #     Deduplicate embeddings across two datasets and return the indices of duplicates between them.
+# #     """
+# #     reach = Reach(vectors=embedding_matrix_1, items=[str(i) for i in range(len(embedding_matrix_1))])
+
+# #     duplicate_indices_in_test = []
+# #     duplicate_to_original_mapping = {}
+
+# #     results = reach.nearest_neighbor_threshold(
+# #         embedding_matrix_2,
+# #         threshold=threshold,
+# #         batch_size=batch_size,
+# #         show_progressbar=True  # Allow internal progress bar
+# #     )
+
+# #     # Process duplicates
+# #     for i, similar_items in enumerate(progress.tqdm(results, desc="Processing duplicates across datasets")):
+# #         similar_indices = [int(item[0]) for item in similar_items if item[1] >= threshold]
+
+# #         if similar_indices:
+# #             duplicate_indices_in_test.append(i)
+# #             duplicate_to_original_mapping[i] = similar_indices[0]
+
+# #     return duplicate_indices_in_test, duplicate_to_original_mapping
+
+# # def display_word_differences(x: str, y: str) -> str:
+# #     diff = ndiff(x.split(), y.split())
+# #     return " ".join([word for word in diff if word.startswith(('+', '-'))])
+
+# # def perform_deduplication(
+# #     deduplication_type,
+# #     dataset1_name,
+# #     dataset1_split,
+# #     dataset1_text_column,
+# #     dataset2_name="",
+# #     dataset2_split="",
+# #     dataset2_text_column="",
+# #     threshold=0.8,
+# #     progress=gr.Progress(track_tqdm=True)
+# # ):
+# #     # Monkey-patch tqdm
+# #     original_tqdm = tqdm.tqdm
+# #     tqdm.tqdm = progress.tqdm
+# #     sys.modules['tqdm'].tqdm = progress.tqdm
+# #     sys.modules['tqdm.auto'].tqdm = progress.tqdm
+
+# #     try:
+# #         # Convert threshold to float
+# #         threshold = float(threshold)
+        
+# #         if deduplication_type == "Single dataset":
+# #             # Check if the dataset is the default one
+# #             if dataset1_name == default_dataset1_name and dataset1_split == default_dataset1_split:
+# #                 ds = ds_default1
+# #             else:
+# #                 ds = load_dataset(dataset1_name, split=dataset1_split)
+            
+# #             # Extract texts
+# #             texts = [example[dataset1_text_column] for example in ds]
+            
+# #             # Compute embeddings
+# #             embedding_matrix = model.encode(texts, show_progressbar=True)  # Enable internal progress bar
+            
+# #             # Deduplicate
+# #             deduplicated_indices, duplicate_to_original_mapping = deduplicate(embedding_matrix, threshold, progress=progress)
+            
+# #             # Prepare the results
+# #             num_duplicates = len(duplicate_to_original_mapping)
+# #             num_total = len(texts)
+# #             num_deduplicated = len(deduplicated_indices)
+            
+# #             result_text = f"**Total documents:** {num_total}\n"
+# #             result_text += f"**Number of duplicates found:** {num_duplicates}\n"
+# #             result_text += f"**Number of unique documents after deduplication:** {num_deduplicated}\n\n"
+            
+# #             # Show deduplicated examples
+# #             result_text += "**Examples of duplicates found:**\n\n"
+# #             num_examples = min(5, num_duplicates)
+# #             for duplicate_idx, original_idx in list(duplicate_to_original_mapping.items())[:num_examples]:
+# #                 original_text = texts[original_idx]
+# #                 duplicate_text = texts[duplicate_idx]
+# #                 differences = display_word_differences(original_text, duplicate_text)
+# #                 result_text += f"**Original text:**\n{original_text}\n\n"
+# #                 result_text += f"**Duplicate text:**\n{duplicate_text}\n\n"
+# #                 result_text += f"**Differences:**\n{differences}\n"
+# #                 result_text += "-" * 50 + "\n\n"
+            
+# #             return result_text
+                
+# #         elif deduplication_type == "Cross-dataset":
+# #             # Dataset 1
+# #             if dataset1_name == default_dataset1_name and dataset1_split == default_dataset1_split:
+# #                 ds1 = ds_default1
+# #             else:
+# #                 ds1 = load_dataset(dataset1_name, split=dataset1_split)
+            
+# #             # Dataset 2
+# #             if dataset2_name == default_dataset2_name and dataset2_split == default_dataset2_split:
+# #                 ds2 = ds_default2
+# #             else:
+# #                 ds2 = load_dataset(dataset2_name, split=dataset2_split)
+            
+# #             # Extract texts
+# #             texts1 = [example[dataset1_text_column] for example in ds1]
+# #             texts2 = [example[dataset2_text_column] for example in ds2]
+            
+# #             # Compute embeddings
+# #             embedding_matrix1 = model.encode(texts1, show_progressbar=True)  # Enable internal progress bar
+# #             embedding_matrix2 = model.encode(texts2, show_progressbar=True)  # Enable internal progress bar
+            
+# #             # Deduplicate across datasets
+# #             duplicate_indices_in_ds2, duplicate_to_original_mapping = deduplicate_across_datasets(embedding_matrix1, embedding_matrix2, threshold, progress=progress)
+            
+# #             num_duplicates = len(duplicate_indices_in_ds2)
+# #             num_total_ds2 = len(texts2)
+# #             num_unique_ds2 = num_total_ds2 - num_duplicates
+            
+# #             result_text = f"**Total documents in {dataset2_name}/{dataset2_split}:** {num_total_ds2}\n"
+# #             result_text += f"**Number of duplicates found in {dataset2_name}/{dataset2_split}:** {num_duplicates}\n"
+# #             result_text += f"**Number of unique documents in {dataset2_name}/{dataset2_split} after deduplication:** {num_unique_ds2}\n\n"
+            
+# #             # Show deduplicated examples
+# #             result_text += "**Examples of duplicates found in Dataset 2:**\n\n"
+# #             num_examples = min(5, num_duplicates)
+# #             for duplicate_idx in duplicate_indices_in_ds2[:num_examples]:
+# #                 original_idx = duplicate_to_original_mapping[duplicate_idx]
+# #                 original_text = texts1[original_idx]
+# #                 duplicate_text = texts2[duplicate_idx]
+# #                 differences = display_word_differences(original_text, duplicate_text)
+# #                 result_text += f"**Original text (Dataset 1):**\n{original_text}\n\n"
+# #                 result_text += f"**Duplicate text (Dataset 2):**\n{duplicate_text}\n\n"
+# #                 result_text += f"**Differences:**\n{differences}\n"
+# #                 result_text += "-" * 50 + "\n\n"
+            
+# #             return result_text
+
+# #     finally:
+# #         # Restore original tqdm
+# #         tqdm.tqdm = original_tqdm
+# #         sys.modules['tqdm'].tqdm = original_tqdm
+# #         sys.modules['tqdm.auto'].tqdm = original_tqdm
+
+# # with gr.Blocks() as demo:
+# #     gr.Markdown("# Semantic Deduplication")
+    
+# #     deduplication_type = gr.Radio(
+# #         choices=["Single dataset", "Cross-dataset"],
+# #         label="Deduplication Type",
+# #         value="Single dataset"
+# #     )
+    
+# #     with gr.Row():
+# #         dataset1_name = gr.Textbox(value="ag_news", label="Dataset 1 Name")
+# #         dataset1_split = gr.Textbox(value="train", label="Dataset 1 Split")
+# #         dataset1_text_column = gr.Textbox(value="text", label="Text Column Name")
+    
+# #     dataset2_inputs = gr.Column(visible=False)
+# #     with dataset2_inputs:
+# #         gr.Markdown("### Dataset 2")
+# #         with gr.Row():
+# #             dataset2_name = gr.Textbox(value="ag_news", label="Dataset 2 Name")
+# #             dataset2_split = gr.Textbox(value="test", label="Dataset 2 Split")
+# #             dataset2_text_column = gr.Textbox(value="text", label="Text Column Name")
+    
+# #     threshold = gr.Slider(
+# #         minimum=0.0,
+# #         maximum=1.0,
+# #         value=0.8,
+# #         label="Similarity Threshold"
+# #     )
+    
+# #     compute_button = gr.Button("Compute")
+    
+# #     output = gr.Markdown()
+    
+# #     # Function to update the visibility of dataset2_inputs
+# #     def update_visibility(deduplication_type_value):
+# #         if deduplication_type_value == "Cross-dataset":
+# #             return gr.update(visible=True)
+# #         else:
+# #             return gr.update(visible=False)
+    
+# #     deduplication_type.change(
+# #         update_visibility,
+# #         inputs=deduplication_type,
+# #         outputs=dataset2_inputs
+# #     )
+    
+# #     compute_button.click(
+# #         fn=perform_deduplication,
+# #         inputs=[
+# #             deduplication_type, 
+# #             dataset1_name, 
+# #             dataset1_split, 
+# #             dataset1_text_column,
+# #             dataset2_name, 
+# #             dataset2_split, 
+# #             dataset2_text_column,
+# #             threshold
+# #         ],
+# #         outputs=output
+# #     )
+    
+# # demo.launch()