Spaces:
Sleeping
Sleeping
feat: add invalid tag filtering
Browse filesAdd filtering for all tags to ensure they are taken from tags list.
Warning will be raised if tags not from tag list are encountered.
app.py
CHANGED
|
@@ -146,7 +146,13 @@ def process_quotes(quotes_file_path: str, quotes_col_name: str, tags_string: str
|
|
| 146 |
for i, quote in enumerate(quotes_data):
|
| 147 |
logger.info(f"Tagging quote {i + 1}/{len(quotes_data)}: {quote}")
|
| 148 |
tags = tag_quote(quote, tags_list)
|
| 149 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 150 |
|
| 151 |
quotes_df['Tags'] = tags_column
|
| 152 |
logger.info("Quotes tagged")
|
|
|
|
| 146 |
for i, quote in enumerate(quotes_data):
|
| 147 |
logger.info(f"Tagging quote {i + 1}/{len(quotes_data)}: {quote}")
|
| 148 |
tags = tag_quote(quote, tags_list)
|
| 149 |
+
valid_tags = []
|
| 150 |
+
for tag in tags: # filter out any hallucinated tags
|
| 151 |
+
if tag in tags_list:
|
| 152 |
+
valid_tags.append(tag)
|
| 153 |
+
else:
|
| 154 |
+
logger.warning(f"Invalid tag {tag} found and has been filtered out.")
|
| 155 |
+
tags_column.append(valid_tags)
|
| 156 |
|
| 157 |
quotes_df['Tags'] = tags_column
|
| 158 |
logger.info("Quotes tagged")
|