kenleeyx commited on
Commit
5024bb9
·
1 Parent(s): 3bf592f

feat: add invalid tag filtering

Browse files

Add filtering for all tags to ensure they are taken from tags list.
Warning will be raised if tags not from tag list are encountered.

Files changed (1) hide show
  1. app.py +7 -1
app.py CHANGED
@@ -146,7 +146,13 @@ def process_quotes(quotes_file_path: str, quotes_col_name: str, tags_string: str
146
  for i, quote in enumerate(quotes_data):
147
  logger.info(f"Tagging quote {i + 1}/{len(quotes_data)}: {quote}")
148
  tags = tag_quote(quote, tags_list)
149
- tags_column.append(tags)
 
 
 
 
 
 
150
 
151
  quotes_df['Tags'] = tags_column
152
  logger.info("Quotes tagged")
 
146
  for i, quote in enumerate(quotes_data):
147
  logger.info(f"Tagging quote {i + 1}/{len(quotes_data)}: {quote}")
148
  tags = tag_quote(quote, tags_list)
149
+ valid_tags = []
150
+ for tag in tags: # filter out any hallucinated tags
151
+ if tag in tags_list:
152
+ valid_tags.append(tag)
153
+ else:
154
+ logger.warning(f"Invalid tag {tag} found and has been filtered out.")
155
+ tags_column.append(valid_tags)
156
 
157
  quotes_df['Tags'] = tags_column
158
  logger.info("Quotes tagged")