Update BERTopic/my_topic_modeling.py

BERTopic/my_topic_modeling.py

@@ -76,13 +76,13 @@ target_col = 'normalized_content'
 
 
 
-def create_logger_file_and_console():
+def create_logger_file_and_console(path_file):
     # create logger for "Sample App"
     logger = logging.getLogger('automated_testing')
     logger.setLevel(logging.DEBUG)
 
     # create file handler which logs even debug messages
-    fileh = logging.FileHandler('info.log', mode='a')
+    fileh = logging.FileHandler(path_file, mode='a')
     fileh.setLevel(logging.DEBUG)
 
     # create console handler with a higher log level
@@ -100,13 +100,13 @@ def create_logger_file_and_console():
 
     return logger
 
-def create_logger_file():
+def create_logger_file(path_file):
     # create logger for "Sample App"
     logger = logging.getLogger('automated_testing')
     logger.setLevel(logging.INFO)
 
     # create file handler which logs even debug messages
-    fileh = logging.FileHandler('info.log', mode='a')
+    fileh = logging.FileHandler(path_file, mode='a')
     fileh.setLevel(logging.INFO)
 
     # create formatter and add it to the handlers
@@ -413,11 +413,17 @@ def coherence_score(topic_model, df_documents):
     return coherence
 
 def working(args: argparse.Namespace, name_dataset: str):
+
+    source = f'en_{name_dataset}'
+    output_subdir_name = source + f'/bertopic2_non_zeroshot_{args.n_topics}topic_'+doc_type+'_'+doc_level+'_'+doc_time
+    output_subdir = os.path.join(output_dir, output_subdir_name)
+    if not os.path.exists(output_subdir):
+        os.makedirs(output_subdir)
     
-    
+    info_log_out = os.path.join(output_subdir, 'info.log')
     ############# Create logger##################################
-    fandc_logger = create_logger_file_and_console()
-    file_logger = create_logger_file()
+    fandc_logger = create_logger_file_and_console(info_log_out)
+    file_logger = create_logger_file(info_log_out)
     console_logger = create_logger_console() 
     ##############################################################
     
@@ -440,11 +446,6 @@ def working(args: argparse.Namespace, name_dataset: str):
     fandc_logger.log(logging.INFO, f'Processing data for {name_dataset} dataset successfully!')
     #######################################################################
     
-    source = f'en_{name_dataset}'
-    output_subdir_name = source + f'/bertopic2_non_zeroshot_{args.n_topics}topic_'+doc_type+'_'+doc_level+'_'+doc_time
-    output_subdir = os.path.join(output_dir, output_subdir_name)
-    if not os.path.exists(output_subdir):
-        os.makedirs(output_subdir)
     
     
     # Create model
@@ -558,6 +559,7 @@ def working(args: argparse.Namespace, name_dataset: str):
 
         # ****** 2
         # Get reduce topics and topic over time for each n_topics
+    if args.need_reduce_n_tops == 'yes':
         fandc_logger.log(logging.INFO, f'Staring reduce topics and topic over time from 10 to 50...')
         for n_topics in [10,20,30,40,50]:
             topic_model_copy = copy.deepcopy(topic_model)