Update BERTopic/my_topic_modeling.py

BERTopic/my_topic_modeling.py

@@ -589,34 +589,7 @@ def working(args: argparse.Namespace, name_dataset: str):
             topic_dtm_path_out = os.path.join(output_subdir, f'topics_dtm_{name_dataset}'+key+'_'+doc_type+'_'+doc_level+'_'+doc_time+'.csv')
             topics_over_time.to_csv(topic_dtm_path_out, encoding='utf-8')
         fandc_logger.log(logging.INFO, f'Save topics over time for {name_dataset} dataset successfully!')
-
-        # ****** 2
-        # Get reduce topics and topic over time for each n_topics
-    if args.need_reduce_n_topics == 'yes':
-        fandc_logger.log(logging.INFO, f'Staring reduce topics and topic over time from 10 to 50...')
-        for n_topics in [10,20,30,40,50]:
-            topic_model_copy = copy.deepcopy(topic_model)
-            topic_model_copy.reduce_topics(df_documents, nr_topics=n_topics)
-            fig = topic_model_copy.visualize_topics(title=f"Intertopic Distance Map: {n_topics} topics")
-            fig.show()
-            vis_save_dir = os.path.join(output_subdir, f'bertopic_reduce_vis_{name_dataset}'+str(n_topics)+'_'+doc_type+'_'+doc_level+'_'+doc_time+'.html')
-            fig.write_html(vis_save_dir)
-
-            topic_info = topic_model_copy.get_topic_info()
-            topic_info_path_out = os.path.join(output_subdir, f'topic_reduce_info_{name_dataset}'+str(n_topics)+'_'+doc_type+'_'+doc_level+'_'+doc_time+'.csv')
-            topic_info.to_csv(topic_info_path_out, encoding='utf-8')
-
-            for key in timestamps_dict.keys():
-                topics_over_time_ = topic_model_copy.topics_over_time(df_documents, timestamps_dict[key])
-                fig = topic_model_copy.visualize_topics_over_time(topics_over_time_, top_n_topics=10, title=f"Topics over time following {key}")
-                fig.show()
-                vis_save_dir = os.path.join(output_subdir, f'bertopic_reduce_dtm_vis_{name_dataset}'+str(n_topics)+'_'+key+'_'+doc_type+'_'+doc_level+'_'+doc_time+'.html')
-                fig.write_html(vis_save_dir)
-
-                topic_dtm_path_out = os.path.join(output_subdir, f'topics_reduce_dtm_{name_dataset}'+str(n_topics)+'_'+key+'_'+doc_type+'_'+doc_level+'_'+doc_time+'.csv')
-                topics_over_time_.to_csv(topic_dtm_path_out, encoding='utf-8')
-
-        fandc_logger.log(logging.INFO, f'Save topics reduce and topic over time for {name_dataset} dataset successfully!')
+    
     ###################################
     fandc_logger.log(logging.INFO, f'ENDING TRAINING TOPIC MODELING {name_dataset} dataset\n')