feat: add random quote retrieval for each tag

-Also add one hot encoding for tags
-Fix bug where app throws error when retained_columns is empty
-Remove limit from ThreadPoolExecutor after determining impact on rate
limit is insignificant

app.py

@@ -9,6 +9,7 @@ from dotenv import load_dotenv # For loading environment variables in local envi
 from collections import Counter # For tabulating tag occurrences
 import logging
 import time
+import random
 from datetime import datetime
 from typing import Generator
 from concurrent.futures import ThreadPoolExecutor, as_completed
@@ -140,8 +141,11 @@ def process_quotes(quotes_file_path: str, quotes_col_name: str, retained_columns
     tags_list = tags_string.split('\n')
     tags_list = [tag.strip() for tag in tags_list]
 
-    retained_cols_list = retained_columns.split(',')
-    retained_cols_list = [colname.strip() for colname in retained_cols_list]
+    if retained_columns:
+        retained_cols_list = retained_columns.split(',')
+        retained_cols_list = [colname.strip() for colname in retained_cols_list]
+    else:
+        retained_cols_list = []
 
     # Transfer quotes data from Excel file into pandas DataFrame, handling potential duplicate column names in the Excel file
     # pd.read_excel will rename duplicates eg foo -> foo.1, causing a mismatch between quotes_col_name and the actual column name
@@ -165,7 +169,7 @@ def process_quotes(quotes_file_path: str, quotes_col_name: str, retained_columns
     tags_results = [None]*len(quotes_data)
 
     # Threading execution of tag_quotes with {max_workers} threads: we send {max_workers} requests to the LLM concurrently.
-    with ThreadPoolExecutor(max_workers=5) as executor:
+    with ThreadPoolExecutor() as executor:
         # Generate futures for each of the quotes and map them to the quote indices
         future_to_index = {
             executor.submit(tag_quote, quote, tags_list): i for i, quote in enumerate(quotes_data)
@@ -184,17 +188,28 @@ def process_quotes(quotes_file_path: str, quotes_col_name: str, retained_columns
             yield (f"Tagged {completed}/{len(quotes_data)} quotes: {quotes_data[i]}", None, None, None)
 
     quotes_df['Tags'] = tags_results
+
+    # One hot encoding of tagged tags
+    for tag in tags_list:
+        quotes_df[tag]=quotes_df['Tags'].apply(lambda quote_tags: int(tag in quote_tags))
     logger.info("Quotes tagged")
 
     # Create hash table of tag occurrences using count_tags function
     tags_counter_df = count_tags(tags_list, quotes_df['Tags'])
-    logger.info("Tags counted")
-    
+
+    # Retrieve 2 quotes at random for each tag and put them in the tags counter DF
+    for tag in tags_counter_df['Tag']:
+        tagged_quotes_list = quotes_df.loc[quotes_df[tag]==1, quotes_col_name].tolist()
+        sample_quotes = random.sample(tagged_quotes_list, min(2, len(tagged_quotes_list)))
+        while len(sample_quotes) < 2:
+            sample_quotes.append(None)
+        [tags_counter_df.loc[tags_counter_df['Tag'] == tag, 'Quote 1'], tags_counter_df.loc[tags_counter_df['Tag'] == tag, 'Quote 2']] = sample_quotes
+        
     #Convert values in tags column from list to str
     quotes_df['Tags'] = quotes_df["Tags"].apply(lambda x: ", ".join(x))
 
     # Return only the quotes column, the new tags column, and any other specified cols to retain
-    output_df = quotes_df[retained_cols_list+[quotes_col_name, 'Tags']] 
+    output_df = quotes_df[retained_cols_list+[quotes_col_name, 'Tags']+tags_list] 
     output_file_path = "output.xlsx"
     with pd.ExcelWriter(output_file_path) as writer:
         output_df.to_excel(writer, sheet_name='Coded Quotes', index=False)