Update utils.py

utils.py

@@ -49,17 +49,17 @@ def update_db_hub(texts, topics, dates):
         dataset = load_dataset(dataset_name, download_config=DownloadConfig(token=api_token))
         # print("Dataset loaded successfully!", dataset)
         # print(dataset)
-        deduplicated_new_dataset = remove_duplicates(
-            dataset['train'], 
-            new_dataset, 
-            unique_key="text", 
-            batch_size=1000  # Adjust batch size based on available memory
-        )
+        # deduplicated_new_dataset = remove_duplicates(
+        #     dataset['train'], 
+        #     new_dataset, 
+        #     unique_key="text", 
+        #     batch_size=1000  # Adjust batch size based on available memory
+        # )
         
-        updated_dataset = concatenate_datasets([dataset['train'], deduplicated_new_dataset])
+        updated_dataset = concatenate_datasets([dataset['train'], new_dataset])
         
         # updated_dataset = new_dataset
-        del dataset
+        # del dataset
     except Exception as e:
         updated_dataset = new_dataset
         print(f"Failed to load dataset: {e}")